Java Code Examples for org.apache.solr.client.solrj.impl.CloudSolrClient#setDefaultCollection()

private DirectSolrInputDocumentWriter createCloudSolrWriter(Context context, Map<String, String> indexConnectionParams)
        throws IOException {
    String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER);
    String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION);

    if (indexZkHost == null) {
        throw new IllegalStateException("No index ZK host defined");
    }

    if (collectionName == null) {
        throw new IllegalStateException("No collection name defined");
    }
    CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build();
    int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(context.getConfiguration());
    solrServer.setZkClientTimeout(zkSessionTimeout);
    solrServer.setZkConnectTimeout(zkSessionTimeout);      
    solrServer.setDefaultCollection(collectionName);

    return new DirectSolrInputDocumentWriter(context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServer);
}
 

private Set<SolrClient> createSolrClients(Map<String, String> indexConnectionParams) throws MalformedURLException {
    String solrMode = getSolrMode(indexConnectionParams);
    if (solrMode.equals("cloud")) {
        String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER);
        String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION);
        CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build();
        int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf());
        solrServer.setZkClientTimeout(zkSessionTimeout);
        solrServer.setZkConnectTimeout(zkSessionTimeout);
        solrServer.setDefaultCollection(collectionName);
        return Collections.singleton((SolrClient)solrServer);
    } else if (solrMode.equals("classic")) {
        PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
        connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams));
        connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams));

        HttpClient httpClient = new DefaultHttpClient(connectionManager);
        return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient));
    } else {
        throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode);
    }

}
 

private static SolrClient cloneCloudSolrClient(SolrClient solrClient, String core) {
	if (VersionUtil.isSolr3XAvailable() || solrClient == null) {
		return null;
	}

	CloudSolrClient cloudServer = (CloudSolrClient) solrClient;
	String zkHost = readField(solrClient, "zkHost");

	Constructor<? extends SolrClient> constructor = (Constructor<? extends SolrClient>) ClassUtils
			.getConstructorIfAvailable(solrClient.getClass(), String.class, LBHttpSolrClient.class);

	CloudSolrClient clone = (CloudSolrClient) BeanUtils.instantiateClass(constructor, zkHost,
			cloneLBHttpSolrClient(cloudServer.getLbClient(), core));

	if (org.springframework.util.StringUtils.hasText(core)) {
		clone.setDefaultCollection(core);
	}
	return clone;
}
 

private SolrClient getSolrClient() throws MalformedURLException {
  if(kerberosAuth) {
    // set kerberos before create SolrClient
    addSecurityProperties();
  }

  if (SolrInstanceAPIType.SINGLE_NODE.toString().equals(this.instanceType)) {
    HttpSolrClient solrClient = new HttpSolrClient.Builder(this.solrURI).build();
    solrClient.setConnectionTimeout(this.connectionTimeout);
    solrClient.setSoTimeout(this.socketTimeout);
    return solrClient;
  } else {
    CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(this.zookeeperConnect).build();
    cloudSolrClient.setDefaultCollection(this.defaultCollection);
    return cloudSolrClient;
  }
}
 

private int indexDocs(CloudSolrClient sourceSolrClient, String collection, int batches) throws IOException, SolrServerException {
  sourceSolrClient.setDefaultCollection(collection);
  int numDocs = 0;
  for (int k = 0; k < batches; k++) {
    UpdateRequest req = new UpdateRequest();
    for (; numDocs < (k + 1) * 100; numDocs++) {
      SolrInputDocument doc = new SolrInputDocument();
      doc.addField("id", "source_" + numDocs);
      doc.addField("xyz", numDocs);
      req.add(doc);
    }
    req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
    req.process(sourceSolrClient);
  }
  log.info("Adding numDocs={}", numDocs);
  return numDocs;
}
 

public List<Map<String, Object>> getAllIndexedDocs(String collection) {
  List<Map<String, Object>> docs = new ArrayList<>();
  CloudSolrClient solr = miniSolrCloudCluster.getSolrClient();
  solr.setDefaultCollection(collection);
  SolrQuery parameters = new SolrQuery();

  // If it's metaalert, we need to adjust the query. We want child docs with the parent,
  // not separate.
  if (collection.equals("metaalert")) {
    parameters.setQuery("source.type:metaalert")
        .setFields("*", "[child parentFilter=source.type:metaalert limit=999]");
  } else {
    parameters.set("q", "*:*");
  }
  try {
    solr.commit();
    QueryResponse response = solr.query(parameters);
    for (SolrDocument solrDocument : response.getResults()) {
      // Use the utils to make sure we get child docs.
      docs.add(SolrUtilities.toDocument(solrDocument).getDocument());
    }
  } catch (SolrServerException | IOException e) {
    e.printStackTrace();
  }
  return docs;
}
 

public SolrClient getSolrServer() {
  if (zkHost != null && zkHost.length() > 0) {
    if (collectionName == null || collectionName.length() == 0) {
      throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config);
    }
    CloudSolrClient cloudSolrClient = new Builder()
        .withZkHost(zkHost)
        .build();
    cloudSolrClient.setDefaultCollection(collectionName);
    cloudSolrClient.setZkClientTimeout(zkClientSessionTimeout); 
    cloudSolrClient.setZkConnectTimeout(zkClientConnectTimeout); 
    return cloudSolrClient;
  } else {
    if (solrUrl == null && solrHomeDir != null) {
      CoreContainer coreContainer = new CoreContainer(solrHomeDir);
      coreContainer.load();
      EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(coreContainer, collectionName);
      return embeddedSolrServer;
    }
    if (solrUrl == null || solrUrl.length() == 0) {
      throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config);
    }
    int solrServerNumThreads = 2;
    int solrServerQueueLength = solrServerNumThreads;
    SolrClient server = new SafeConcurrentUpdateSolrServer(solrUrl, solrServerQueueLength, solrServerNumThreads);
    return server;
  }
}
 

private CloudSolrClient createSolrCloudClient(String zkConnectionString, String collection) {
  logger.info("Using zookeepr. zkConnectString=" + zkConnectionString);
  final ZkConnection zkConnection = createZKConnection(zkConnectionString);
  final CloudSolrClient.Builder builder =
    new CloudSolrClient.Builder(zkConnection.getZkHosts(), Optional.ofNullable(zkConnection.getZkChroot()));
  CloudSolrClient solrClient = builder.build();
  solrClient.setDefaultCollection(collection);
  return solrClient;
}
 

public void addDocs(String collection, List<Map<String, Object>> docs)
    throws IOException, SolrServerException {
  CloudSolrClient solr = miniSolrCloudCluster.getSolrClient();
  solr.setDefaultCollection(collection);
  Collection<SolrInputDocument> solrInputDocuments = docs.stream().map(doc -> {
    SolrInputDocument solrInputDocument = new SolrInputDocument();
    for (Entry<String, Object> entry : doc.entrySet()) {
      // If the entry itself is a map, add it as a child document. Handle one level of nesting.
      if (entry.getValue() instanceof List && !entry.getKey().equals(
          MetaAlertConstants.METAALERT_FIELD)) {
        for (Object entryItem : (List)entry.getValue()) {
          if (entryItem instanceof Map) {
            @SuppressWarnings("unchecked")
            Map<String, Object> childDoc = (Map<String, Object>) entryItem;
            SolrInputDocument childInputDoc = new SolrInputDocument();
            for (Entry<String, Object> childEntry : childDoc.entrySet()) {
              childInputDoc.addField(childEntry.getKey(), childEntry.getValue());
            }
            solrInputDocument.addChildDocument(childInputDoc);
          }
        }
      } else {
        solrInputDocument.addField(entry.getKey(), entry.getValue());
      }
    }
    return solrInputDocument;
  }).collect(Collectors.toList());

  checkUpdateResponse(solr.add(collection, solrInputDocuments));
  // Make sure to commit so things show up
  checkUpdateResponse(solr.commit(true, true));
}
 

@BeforeClass
public static void setupCluster() throws Exception {
  
  differentUniqueId = random().nextBoolean();
  
  final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");

  String configName = "solrCloudCollectionConfig";
  int nodeCount = 5;
  configureCluster(nodeCount)
     .addConfig(configName, configDir)
     .configure();
  
  Map<String, String> collectionProperties = new HashMap<>();
  collectionProperties.put("config", "solrconfig-doctransformers.xml" );
  collectionProperties.put("schema", "schema-docValuesJoin.xml"); 

  int shards = 2;
  int replicas = 2 ;
  CollectionAdminRequest.createCollection(people, configName, shards, replicas)
      .withProperty("config", "solrconfig-doctransformers.xml")
      .withProperty("schema", "schema-docValuesJoin.xml")
      .process(cluster.getSolrClient());

  CollectionAdminRequest.createCollection(depts, configName, shards, replicas)
      .withProperty("config", "solrconfig-doctransformers.xml")
      .withProperty("schema", 
            differentUniqueId ? "schema-minimal-with-another-uniqkey.xml":
                                "schema-docValuesJoin.xml")
      .process(cluster.getSolrClient());

  CloudSolrClient client = cluster.getSolrClient();
  client.setDefaultCollection(people);
  
  ZkStateReader zkStateReader = client.getZkStateReader();
  AbstractDistribZkTestBase.waitForRecoveriesToFinish(people, zkStateReader, true, true, 30);
  
  AbstractDistribZkTestBase.waitForRecoveriesToFinish(depts, zkStateReader, false, true, 30);
}
 

/**
 * Creates a new 2x2 collection using a unique name, blocking until it's state is fully active, 
 * and sets that collection as the default on the cluster's default CloudSolrClient.
 * 
 * @return the name of the new collection
 */
public static String createAndSetNewDefaultCollection() throws Exception {
  final CloudSolrClient cloudClient = cluster.getSolrClient();
  final String name = "test_collection_" + NAME_COUNTER.getAndIncrement();
  assertEquals(RequestStatusState.COMPLETED,
               CollectionAdminRequest.createCollection(name, "_default", 2, 2)
               .processAndWait(cloudClient, DEFAULT_TIMEOUT));
  cloudClient.waitForState(name, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
                           (n, c) -> DocCollection.isFullyActive(n, c, 2, 2));
  cloudClient.setDefaultCollection(name);
  return name;
}
 

CloudSolrClient createCollection(String collectionName, int repFactor) throws Exception {

      CollectionAdminRequest
          .createCollection(collectionName, "conf", 1, repFactor)
          .setMaxShardsPerNode(100)
          .process(cluster.getSolrClient());

    cluster.waitForActiveCollection(collectionName, 1, repFactor);

    CloudSolrClient client = cluster.getSolrClient();
    client.setDefaultCollection(collectionName);
    return client;
  }
 

public static CloudSolrClient createSolrClient() {
    CloudSolrClient solrClient = null;

    try {
        solrClient = new CloudSolrClient.Builder(Collections.singletonList(SOLR_LOCATION), Optional.empty()).build();
        solrClient.setDefaultCollection(SOLR_COLLECTION);
    } catch (Exception e) {
        e.printStackTrace();
    }
    return solrClient;
}
 

/**
 * This test start cdcr source, adds data,starts target cluster, verifies replication,
 * stops cdcr replication and buffering, adds more data, re-enables cdcr and verify replication
 */
public void testBootstrapWithSourceCluster() throws Exception {
  // start the target first so that we know its zkhost
  MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));
  try {
    System.out.println("Target zkHost = " + target.getZkServer().getZkAddress());
    System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress());

    MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr"));
    try {
      source.uploadConfigSet(configset("cdcr-source"), "cdcr-source");

      CollectionAdminRequest.createCollection("cdcr-source", "cdcr-source", 1, 1)
          .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
          .process(source.getSolrClient());
      source.waitForActiveCollection("cdcr-source", 1, 1);

      CloudSolrClient sourceSolrClient = source.getSolrClient();
      int docs = (TEST_NIGHTLY ? 100 : 10);
      int numDocs = indexDocs(sourceSolrClient, "cdcr-source", docs);

      QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*"));
      assertEquals("", numDocs, response.getResults().getNumFound());

      // setup the target cluster
      target.uploadConfigSet(configset("cdcr-target"), "cdcr-target");
      CollectionAdminRequest.createCollection("cdcr-target", "cdcr-target", 1, 1)
          .process(target.getSolrClient());
      target.waitForActiveCollection("cdcr-target", 1, 1);
      CloudSolrClient targetSolrClient = target.getSolrClient();
      targetSolrClient.setDefaultCollection("cdcr-target");

      CdcrTestsUtil.cdcrStart(targetSolrClient);
      CdcrTestsUtil.cdcrStart(sourceSolrClient);

      response = CdcrTestsUtil.getCdcrQueue(sourceSolrClient);
      if (log.isInfoEnabled()) {
        log.info("Cdcr queue response: {}", response.getResponse());
      }
      long foundDocs = CdcrTestsUtil.waitForClusterToSync(numDocs, targetSolrClient);
      assertEquals("Document mismatch on target after sync", numDocs, foundDocs);

      int total_tlogs_in_index = FSDirectory.open(target.getBaseDir().resolve("node1").
          resolve("cdcr-target_shard1_replica_n1").resolve("data").
          resolve("tlog")).listAll().length;

      assertEquals("tlogs count should be ZERO",0, total_tlogs_in_index);

      CdcrTestsUtil.cdcrStop(sourceSolrClient);
      CdcrTestsUtil.cdcrDisableBuffer(sourceSolrClient);

      int c = 0;
      for (int k = 0; k < 10; k++) {
        UpdateRequest req = new UpdateRequest();
        for (; c < (k + 1) * 100; c++, numDocs++) {
          SolrInputDocument doc = new SolrInputDocument();
          doc.addField("id", "source_" + numDocs);
          doc.addField("xyz", numDocs);
          req.add(doc);
        }
        req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
        log.info("Adding 100 docs with commit=true, numDocs={}", numDocs);
        req.process(sourceSolrClient);
      }

      response = sourceSolrClient.query(new SolrQuery("*:*"));
      assertEquals("", numDocs, response.getResults().getNumFound());

      CdcrTestsUtil.cdcrStart(sourceSolrClient);
      CdcrTestsUtil.cdcrEnableBuffer(sourceSolrClient);

      foundDocs = CdcrTestsUtil.waitForClusterToSync(numDocs, targetSolrClient);
      assertEquals("Document mismatch on target after sync", numDocs, foundDocs);

    } finally {
      source.shutdown();
    }
  } finally {
    target.shutdown();
  }
}
 

public static CloudSolrClient createCloudClient(MiniSolrCloudCluster cluster, String defaultCollection) {
  CloudSolrClient server = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean());
  if (defaultCollection != null) server.setDefaultCollection(defaultCollection);
  return server;
}
 

protected CloudSolrClient createCloudClient(String defaultCollection) {
  CloudSolrClient server = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean());
  if (defaultCollection != null) server.setDefaultCollection(defaultCollection);
  return server;
}
 

@Test
public void test() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();
  solrClient.setDefaultCollection(COLLECTION_NAME);
  for (int i=0; i<10; i++)  {
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", String.valueOf(i));
    solrClient.add(doc);
  }
  solrClient.commit();

  // find the leader node
  Replica replica = solrClient.getZkStateReader().getLeaderRetry(COLLECTION_NAME, "shard1");
  JettySolrRunner replicaJetty = null;
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (JettySolrRunner jettySolrRunner : jettySolrRunners) {
    int port = jettySolrRunner.getBaseUrl().getPort();
    if (replica.getStr(BASE_URL_PROP).contains(":" + port))  {
      replicaJetty = jettySolrRunner;
      break;
    }
  }

  // kill the leader
  replicaJetty.stop();

  // add a replica (asynchronously)
  CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, "shard1");
  String asyncId = addReplica.processAsync(solrClient);

  // wait a bit
  Thread.sleep(1000);

  // bring the old leader node back up
  replicaJetty.start();

  // wait until everyone is active
  solrClient.waitForState(COLLECTION_NAME, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
      (n, c) -> DocCollection.isFullyActive(n, c, 1, 2));

  // now query each replica and check for consistency
  assertConsistentReplicas(solrClient, solrClient.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME).getSlice("shard1"));

  // sanity check that documents still exist
  QueryResponse response = solrClient.query(new SolrQuery("*:*"));
  assertEquals("Indexed documents not found", 10, response.getResults().getNumFound());
}
 

@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
@Test
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028")
public void testBootstrapWithContinousIndexingOnSourceCluster() throws Exception {
  // start the target first so that we know its zkhost
  MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));
  try {
    if (log.isInfoEnabled()) {
      log.info("Target zkHost = {}", target.getZkServer().getZkAddress());
    }
    System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress());

    MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr"));
    try {
      source.uploadConfigSet(configset("cdcr-source"), "cdcr-source");

      CollectionAdminRequest.createCollection("cdcr-source", "cdcr-source", 1, 1)
          .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
          .process(source.getSolrClient());
      source.waitForActiveCollection("cdcr-source", 1, 1);
      CloudSolrClient sourceSolrClient = source.getSolrClient();
      int docs = (TEST_NIGHTLY ? 100 : 10);
      int numDocs = indexDocs(sourceSolrClient, "cdcr-source", docs);

      QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*"));
      assertEquals("", numDocs, response.getResults().getNumFound());

      // setup the target cluster
      target.uploadConfigSet(configset("cdcr-target"), "cdcr-target");
      CollectionAdminRequest.createCollection("cdcr-target", "cdcr-target", 1, 1)
          .process(target.getSolrClient());
      target.waitForActiveCollection("cdcr-target", 1, 1);
      CloudSolrClient targetSolrClient = target.getSolrClient();
      targetSolrClient.setDefaultCollection("cdcr-target");
      Thread.sleep(1000);

      CdcrTestsUtil.cdcrStart(targetSolrClient);
      CdcrTestsUtil.cdcrStart(sourceSolrClient);
      int c = 0;
      for (int k = 0; k < docs; k++) {
        UpdateRequest req = new UpdateRequest();
        for (; c < (k + 1) * 100; c++, numDocs++) {
          SolrInputDocument doc = new SolrInputDocument();
          doc.addField("id", "source_" + numDocs);
          doc.addField("xyz", numDocs);
          req.add(doc);
        }
        req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
        log.info("Adding {} docs with commit=true, numDocs={}", docs, numDocs);
        req.process(sourceSolrClient);
      }

      response = sourceSolrClient.query(new SolrQuery("*:*"));
      assertEquals("", numDocs, response.getResults().getNumFound());

      response = CdcrTestsUtil.getCdcrQueue(sourceSolrClient);
      if (log.isInfoEnabled()) {
        log.info("Cdcr queue response: {}", response.getResponse());
      }
      long foundDocs = CdcrTestsUtil.waitForClusterToSync(numDocs, targetSolrClient);
      assertEquals("Document mismatch on target after sync", numDocs, foundDocs);

    } finally {
      source.shutdown();
    }
  } finally {
    target.shutdown();
  }
}
 

protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) {
  CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean(), 30000, socketTimeout);
  if (defaultCollection != null) client.setDefaultCollection(defaultCollection);
  return client;
}
 

@Override
public SolrClient getInstance() {

    Logger log = LoggerFactory.getLogger(SolrServerProvider.class);

    String host = SearchConfiguration.get(SearchConfiguration.SERVER_HOST);
    //Backwards compatibility
    String solrHost = SearchConfiguration.get(SearchConfiguration.SERVER_SOLR_HOST);
    if(host == null & solrHost != null) {
        host = solrHost;
    }

    if(host == null) {
        log.error("{} has to be set", SearchConfiguration.SERVER_HOST);
        throw new RuntimeException(SearchConfiguration.SERVER_HOST + " has to be set");
    }

    String collection = SearchConfiguration.get(SearchConfiguration.SERVER_COLLECTION);
    //Backwards compatibility
    String solrCollection = SearchConfiguration.get(SearchConfiguration.SERVER_SOLR_COLLECTION);
    if(collection == null & solrCollection != null) {
        collection = solrCollection;
    }

    final String connectionTimeout = SearchConfiguration.get(SearchConfiguration.SERVER_CONNECTION_TIMEOUT);
    final String soTimeout = SearchConfiguration.get(SearchConfiguration.SERVER_SO_TIMEOUT);

    if(SearchConfiguration.get(SearchConfiguration.SERVER_SOLR_CLOUD, false)) {
        log.info("Instantiating solr cloud client: {}", host);

        if(collection != null) {
            CloudSolrClient client = new CloudSolrClient.Builder(Arrays.asList(host), Optional.empty()).build();
            client.setDefaultCollection(collection);

            if(StringUtils.isNotEmpty(connectionTimeout)) {
                client.setZkConnectTimeout(Integer.valueOf(connectionTimeout));
            }

            if(StringUtils.isNotEmpty(soTimeout)) {
                client.setZkClientTimeout(Integer.valueOf(soTimeout));
            }

            return client;
        } else {
            log.error(SearchConfiguration.SERVER_COLLECTION + " has to be set");
            throw new RuntimeException(SearchConfiguration.SERVER_COLLECTION + " has to be set");
        }

    } else {

        if(collection != null) {
            host = String.join("/",host,collection);
        }
        log.info("Instantiating solr http client: {}", host);
        return new HttpSolrClient.Builder(host).build();

    }

}