Java Code Examples for org.apache.solr.client.solrj.impl.CloudSolrClient#connect()

The following examples show how to use org.apache.solr.client.solrj.impl.CloudSolrClient#connect() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractSolrConfigHandler.java    From ambari-logsearch with Apache License 2.0 6 votes vote down vote up
@Override
public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
  boolean reloadCollectionNeeded = false;
  String separator = FileSystems.getDefault().getSeparator();
  solrClient.connect();
  SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
  try {
    ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
    boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName());
    if (configExists) {
      uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName());
      reloadCollectionNeeded = doIfConfigExists(solrPropsConfig, zkClient, separator);
    } else {
      doIfConfigNotExist(solrPropsConfig, zkConfigManager);
      uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName());
    }
  } catch (Exception e) {
    throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)",
      solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e);
  }
  return reloadCollectionNeeded;
}
 
Example 2
Source File: SolrClientFactory.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public CloudSolrClient createCloudSolrClient(String zookeeperConnectionString) {
  NoOpResponseParser responseParser = new NoOpResponseParser();
  responseParser.setWriterType("json");

  ConnectStringParser parser = new ConnectStringParser(zookeeperConnectionString);

  CloudSolrClient.Builder cloudBuilder = new CloudSolrClient.Builder(
      parser.getServerAddresses().stream()
          .map(address -> String.format(Locale.ROOT, "%s:%s", address.getHostString(), address.getPort()))
          .collect(Collectors.toList()),
      Optional.ofNullable(parser.getChrootPath()));

  cloudBuilder.withConnectionTimeout(settings.getHttpConnectionTimeout())
      .withSocketTimeout(settings.getHttpReadTimeout());

  CloudSolrClient client = cloudBuilder.build();
  client.setParser(responseParser);

  client.connect();

  return client;
}
 
Example 3
Source File: BaseCdcrDistributedZkTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Asserts that the collection has the correct number of shards and replicas
 */
protected void assertCollectionExpectations(String collectionName) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ClusterState clusterState = client.getZkStateReader().getClusterState();

    assertTrue("Could not find new collection " + collectionName, clusterState.hasCollection(collectionName));
    Map<String, Slice> shards = clusterState.getCollection(collectionName).getSlicesMap();
    // did we find expectedSlices shards/shards?
    assertEquals("Found new collection " + collectionName + ", but mismatch on number of shards.", shardCount, shards.size());
    int totalShards = 0;
    for (String shardName : shards.keySet()) {
      totalShards += shards.get(shardName).getReplicas().size();
    }
    int expectedTotalShards = shardCount * replicationFactor;
    assertEquals("Found new collection " + collectionName + " with correct number of shards, but mismatch on number " +
        "of shards.", expectedTotalShards, totalShards);
  } finally {
    client.close();
  }
}
 
Example 4
Source File: TestSolrCloudClusterSupport.java    From storm-solr with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void startCluster() throws Exception {
  File solrXml = new File("src/test/resources/solr.xml");

  tempDir = Files.createTempDirectory("MiniSolrCloudCluster");

  try {
    cluster = new MiniSolrCloudCluster(1, null, tempDir, MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML, null, null);
  } catch (Exception exc) {
    log.error("Failed to initialize a MiniSolrCloudCluster due to: " + exc, exc);
    throw exc;
  }

  cloudSolrClient = new CloudSolrClient(cluster.getZkServer().getZkAddress(), true);
  cloudSolrClient.connect();

  assertTrue(!cloudSolrClient.getZkStateReader().getClusterState().getLiveNodes().isEmpty());
}
 
Example 5
Source File: Solr6Index.java    From atlas with Apache License 2.0 5 votes vote down vote up
private SolrClient createSolrClient() {
    if(logger.isDebugEnabled()) {
        logger.debug("HttpClientBuilder = {}", HttpClientUtil.getHttpClientBuilder(), new Exception());
    }
    final ModifiableSolrParams clientParams = new ModifiableSolrParams();
    SolrClient solrClient = null;

    Mode mode = Mode.parse(configuration.get(SOLR_MODE));
    switch (mode) {
        case CLOUD:
            final CloudSolrClient cloudServer = new CloudSolrClient.Builder()
                    .withLBHttpSolrClientBuilder(
                            new LBHttpSolrClient.Builder()
                                    .withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams))
                                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    )
                    .withZkHost(getZookeeperURLs(configuration))
                    .sendUpdatesOnlyToShardLeaders()
                    .build();
            cloudServer.connect();
            solrClient = cloudServer;
            logger.info("Created solr client using Cloud based configuration.");
            break;
        case HTTP:
            clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, configuration.get(HTTP_ALLOW_COMPRESSION).toString());
            clientParams.add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, configuration.get(HTTP_CONNECTION_TIMEOUT).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, configuration.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS, configuration.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
            final HttpClient client = HttpClientUtil.createClient(clientParams);
            solrClient = new LBHttpSolrClient.Builder()
                    .withHttpClient(client)
                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    .build();
            logger.info("Created solr client using HTTP based configuration.");
            break;
        default:
            throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
    }
    return solrClient;
}
 
Example 6
Source File: SolrCLI.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected void waitToSeeLiveNodes(int maxWaitSecs, String zkHost, int numNodes) {
  CloudSolrClient cloudClient = null;
  try {
    cloudClient = new CloudSolrClient.Builder(Collections.singletonList(zkHost), Optional.empty())
        .build();
    cloudClient.connect();
    Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
    int numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0;
    long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWaitSecs, TimeUnit.SECONDS);
    while (System.nanoTime() < timeout && numLiveNodes < numNodes) {
      echo("\nWaiting up to "+maxWaitSecs+" seconds to see "+
          (numNodes-numLiveNodes)+" more nodes join the SolrCloud cluster ...");
      try {
        Thread.sleep(2000);
      } catch (InterruptedException ie) {
        Thread.interrupted();
      }
      liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
      numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0;
    }
    if (numLiveNodes < numNodes) {
      echo("\nWARNING: Only "+numLiveNodes+" of "+numNodes+
          " are active in the cluster after "+maxWaitSecs+
          " seconds! Please check the solr.log for each node to look for errors.\n");
    }
  } catch (Exception exc) {
    CLIO.err("Failed to see if "+numNodes+" joined the SolrCloud cluster due to: "+exc);
  } finally {
    if (cloudClient != null) {
      try {
        cloudClient.close();
      } catch (Exception ignore) {}
    }
  }
}
 
Example 7
Source File: BaseCdcrDistributedZkTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void waitForCollectionToDisappear(String collection) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ZkStateReader zkStateReader = client.getZkStateReader();
    AbstractDistribZkTestBase.waitForCollectionToDisappear(collection, zkStateReader, true, 15);
  } finally {
    client.close();
  }
}
 
Example 8
Source File: BaseCdcrDistributedZkTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void waitForRecoveriesToFinish(String collection, boolean verbose) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ZkStateReader zkStateReader = client.getZkStateReader();
    super.waitForRecoveriesToFinish(collection, zkStateReader, verbose);
  } finally {
    client.close();
  }
}
 
Example 9
Source File: SolrCloudScraperTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private SolrCloudScraper createSolrCloudScraper() {
  CloudSolrClient solrClient = new CloudSolrClient.Builder(
      Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty())
      .build();

  NoOpResponseParser responseParser = new NoOpResponseParser();
  responseParser.setWriterType("json");

  solrClient.setParser(responseParser);

  solrClient.connect();

  SolrClientFactory factory = new SolrClientFactory(PrometheusExporterSettings.builder().build());

  return new SolrCloudScraper(solrClient, executor, factory);
}
 
Example 10
Source File: BaseCdcrDistributedZkTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/**
 * Updates the mappings between the jetty's instances and the zookeeper cluster state.
 */
protected void updateMappingsFromZk(String collection) throws Exception {
  List<CloudJettyRunner> cloudJettys = new ArrayList<>();
  Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
  Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();

  CloudSolrClient cloudClient = this.createCloudClient(null);
  try {
    cloudClient.connect();
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(collection);

    for (JettySolrRunner jetty : jettys) {
      int port = jetty.getLocalPort();
      if (port == -1) {
        throw new RuntimeException("Cannot find the port for jetty");
      }

      nextJetty:
      for (Slice shard : coll.getSlices()) {
        Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
        for (Map.Entry<String, Replica> entry : entries) {
          Replica replica = entry.getValue();
          if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
            if (!shardToJetty.containsKey(shard.getName())) {
              shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
            }
            boolean isLeader = shard.getLeader() == replica;
            CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
            shardToJetty.get(shard.getName()).add(cjr);
            if (isLeader) {
              shardToLeaderJetty.put(shard.getName(), cjr);
            }
            cloudJettys.add(cjr);
            break nextJetty;
          }
        }
      }
    }

    List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
    if (oldRunners != null)  {
      // must close resources for the old entries
      for (CloudJettyRunner oldRunner : oldRunners) {
        IOUtils.closeQuietly(oldRunner.client);
      }
    }

    this.cloudJettys.put(collection, cloudJettys);
    this.shardToJetty.put(collection, shardToJetty);
    this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
  } finally {
    cloudClient.close();
  }
}
 
Example 11
Source File: MtasSolrTestDistributedSearchConsistency.java    From mtas with Apache License 2.0 4 votes vote down vote up
/**
 * Creates the cloud.
 */
private static void createCloud() {
  Path dataPath = Paths.get("src" + File.separator + "test" + File.separator
      + "resources" + File.separator + "data");
  String solrxml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML;
  JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").build();
  File cloudBase = Files.createTempDir();
  cloudBaseDir = cloudBase.toPath();
  // create subdirectories
  Path clusterDir = cloudBaseDir.resolve("cluster");
  Path logDir = cloudBaseDir.resolve("log");
  if (clusterDir.toFile().mkdir() && logDir.toFile().mkdir()) {
    // set log directory
    System.setProperty("solr.log.dir", logDir.toAbsolutePath().toString());
    try {
      cloudCluster = new MiniSolrCloudCluster(1, clusterDir, solrxml,
          jettyConfig);
      CloudSolrClient client = cloudCluster.getSolrClient();
      client.connect();
      createCloudCollection(COLLECTION_ALL_OPTIMIZED, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_ALL_MULTIPLE_SEGMENTS, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_PART1_OPTIMIZED, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_PART2_MULTIPLE_SEGMENTS, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_DISTRIBUTED, 1, 1,
          dataPath.resolve("conf"));

      // collection1
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(1));
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(2));
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(3));
      client.commit(COLLECTION_ALL_OPTIMIZED);
      client.optimize(COLLECTION_ALL_OPTIMIZED);
      // collection2
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(1));
      client.commit(COLLECTION_ALL_MULTIPLE_SEGMENTS);
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(2));
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(3));
      client.commit(COLLECTION_ALL_MULTIPLE_SEGMENTS);
      // collection3
      client.add(COLLECTION_PART1_OPTIMIZED, solrDocuments.get(1));
      client.commit(COLLECTION_PART1_OPTIMIZED);
      // collection4
      client.add(COLLECTION_PART2_MULTIPLE_SEGMENTS, solrDocuments.get(2));
      client.add(COLLECTION_PART2_MULTIPLE_SEGMENTS, solrDocuments.get(3));
      client.commit(COLLECTION_PART2_MULTIPLE_SEGMENTS);
    } catch (Exception e) {
      e.printStackTrace();
      log.error(e);
    }
  } else {
    log.error("couldn't create directories");
  }
}
 
Example 12
Source File: ChronixSolrCloudStorage.java    From chronix.spark with Apache License 2.0 4 votes vote down vote up
/**
 * Returns the list of shards of the default collection.
 *
 * @param zkHost            ZooKeeper URL
 * @param chronixCollection Solr collection name for chronix time series data
 * @return the list of shards of the default collection
 */
public List<String> getShardList(String zkHost, String chronixCollection) throws IOException {

    CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost);
    List<String> shards = new ArrayList<>();

    try {
        cloudSolrClient.connect();

        ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

        ClusterState clusterState = zkStateReader.getClusterState();

        String[] collections;
        if (clusterState.hasCollection(chronixCollection)) {
            collections = new String[]{chronixCollection};
        } else {
            // might be a collection alias?
            Aliases aliases = zkStateReader.getAliases();
            String aliasedCollections = aliases.getCollectionAlias(chronixCollection);
            if (aliasedCollections == null)
                throw new IllegalArgumentException("Collection " + chronixCollection + " not found!");
            collections = aliasedCollections.split(",");
        }

        Set<String> liveNodes = clusterState.getLiveNodes();
        Random random = new Random(5150);

        for (String coll : collections) {
            for (Slice slice : clusterState.getSlices(coll)) {
                List<String> replicas = new ArrayList<>();
                for (Replica r : slice.getReplicas()) {
                    if (r.getState().equals(Replica.State.ACTIVE)) {
                        ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
                        if (liveNodes.contains(replicaCoreProps.getNodeName()))
                            replicas.add(replicaCoreProps.getCoreUrl());
                    }
                }
                int numReplicas = replicas.size();
                if (numReplicas == 0)
                    throw new IllegalStateException("Shard " + slice.getName() + " in collection " +
                            coll + " does not have any active replicas!");

                String replicaUrl = (numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
                shards.add(replicaUrl);
            }
        }
    } finally {
        cloudSolrClient.close();
    }

    return shards;
}