Java Code Examples for org.apache.solr.client.solrj.impl.CloudSolrClient#commit()

public List<Map<String, Object>> getAllIndexedDocs(String collection) {
  List<Map<String, Object>> docs = new ArrayList<>();
  CloudSolrClient solr = miniSolrCloudCluster.getSolrClient();
  solr.setDefaultCollection(collection);
  SolrQuery parameters = new SolrQuery();

  // If it's metaalert, we need to adjust the query. We want child docs with the parent,
  // not separate.
  if (collection.equals("metaalert")) {
    parameters.setQuery("source.type:metaalert")
        .setFields("*", "[child parentFilter=source.type:metaalert limit=999]");
  } else {
    parameters.set("q", "*:*");
  }
  try {
    solr.commit();
    QueryResponse response = solr.query(parameters);
    for (SolrDocument solrDocument : response.getResults()) {
      // Use the utils to make sure we get child docs.
      docs.add(SolrUtilities.toDocument(solrDocument).getDocument());
    }
  } catch (SolrServerException | IOException e) {
    e.printStackTrace();
  }
  return docs;
}
 

private int indexDocs(String collectionName, boolean useUUID) throws Exception {
  Random random = new Random(docsSeed);// use a constant seed for the whole test run so that we can easily re-index.
  int numDocs = random.nextInt(100);
  if (numDocs == 0) {
    log.info("Indexing ZERO test docs");
    return 0;
  }

  List<SolrInputDocument> docs = new ArrayList<>(numDocs);
  for (int i=0; i<numDocs; i++) {
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", ((useUUID == true) ? java.util.UUID.randomUUID().toString() : i));
    doc.addField("shard_s", "shard" + (1 + random.nextInt(NUM_SHARDS))); // for implicit router
    docs.add(doc);
  }

  CloudSolrClient client = cluster.getSolrClient();
  client.add(collectionName, docs); //batch
  client.commit(collectionName);

  log.info("Indexed {} docs to collection: {}", numDocs, collectionName);

  return numDocs;
}
 

private void testModifyField(String collection) throws IOException, SolrServerException {
  CloudSolrClient cloudClient = cluster.getSolrClient();

  SolrInputDocument doc = new SolrInputDocument("id", "3");
  cloudClient.add(collection, doc);
  cloudClient.commit(collection);

  String fieldName = "id";
  SchemaRequest.Field getFieldRequest = new SchemaRequest.Field(fieldName);
  SchemaResponse.FieldResponse getFieldResponse = getFieldRequest.process(cloudClient, collection);
  Map<String, Object> field = getFieldResponse.getField();
  field.put("docValues", true);
  SchemaRequest.ReplaceField replaceRequest = new SchemaRequest.ReplaceField(field);
  SchemaResponse.UpdateResponse replaceResponse = replaceRequest.process(cloudClient, collection);
  assertNull(replaceResponse.getResponse().get("errors"));
  CollectionAdminRequest.Reload reloadRequest = CollectionAdminRequest.reloadCollection(collection);
  CollectionAdminResponse response = reloadRequest.process(cloudClient);
  assertEquals(0, response.getStatus());
  assertTrue(response.isSuccess());

}
 

/**
 * Invokes a commit on the given collection.
 */
protected void commit(String collection) throws IOException, SolrServerException {
  CloudSolrClient client = createCloudClient(collection);
  try {
    client.commit(true, true);
  } finally {
    client.close();
  }
}
 

@BeforeClass
public static void setup() throws IOException, SolrServerException {
    CloudSolrClient solrClient = createSolrClient();
    Path currentDir = Paths.get(ZK_CONFIG_PATH);
    ZkClientClusterStateProvider stateProvider = new ZkClientClusterStateProvider(SOLR_LOCATION);
    stateProvider.uploadConfig(currentDir, ZK_CONFIG_NAME);
    solrClient.setDefaultCollection(SOLR_COLLECTION);

    if (!solrClient.getZkStateReader().getClusterState().hasCollection(SOLR_COLLECTION)) {
        CollectionAdminRequest.Create createCollection = CollectionAdminRequest.createCollection(SOLR_COLLECTION, ZK_CONFIG_NAME, 1, 1);
        createCollection.process(solrClient);
    } else {
        solrClient.deleteByQuery("*:*");
    }

    for (int i = 0; i < 10; i++) {
        SolrInputDocument doc = new SolrInputDocument();
        doc.addField("id", "doc" + i);
        Date date = new Date();
        doc.addField("created", DATE_FORMAT.format(date));
        doc.addField("string_single", "single" + i + ".1");
        doc.addField("string_multi", "multi" + i + ".1");
        doc.addField("string_multi", "multi" + i + ".2");
        doc.addField("integer_single", i);
        doc.addField("integer_multi", 1);
        doc.addField("integer_multi", 2);
        doc.addField("integer_multi", 3);
        doc.addField("double_single", 0.5 + i);

        solrClient.add(doc);
    }
    solrClient.commit();
}
 

@Before
public void clean() throws IOException, SolrServerException {
  CloudSolrClient client = cluster.getSolrClient();
  client.deleteByQuery("*:*");
  client.commit();
  resetFields(fieldsToTestSingle);
  resetFields(fieldsToTestMulti);
  resetFields(fieldsToTestGroupSortFirst);
  resetFields(fieldsToTestGroupSortLast);

}
 

void addDocuments(CloudSolrClient cloudSolrClient,
    int numCommits, int numDocsPerCommit, boolean optimize) throws Exception {
  for (int cc = 1; cc <= numCommits; ++cc) {
    for (int nn = 1; nn <= numDocsPerCommit; ++nn) {
      ++numDocs;
      final Integer docKey = numDocs;
      SolrInputDocument doc = new SolrInputDocument();
      doc.setField(KEY_FIELD, ""+docKey);
      final int MM = rand.nextInt(60); // minutes
      if (minTimestampMM == null || MM <= minTimestampMM.intValue()) {
        if (minTimestampMM != null && MM < minTimestampMM.intValue()) {
          minTimestampDocKeys.clear();
        }
        minTimestampMM = MM;
        minTimestampDocKeys.add(docKey);
      }
      if (maxTimestampMM == null || maxTimestampMM.intValue() <= MM) {
        if (maxTimestampMM != null && maxTimestampMM.intValue() < MM) {
          maxTimestampDocKeys.clear();
        }
        maxTimestampMM = MM;
        maxTimestampDocKeys.add(docKey);
      }
      doc.setField(TIMESTAMP_FIELD, MM);
      doc.setField(ODD_FIELD, ""+(numDocs % 2));
      doc.setField(QUAD_FIELD, ""+(numDocs % 4)+1);
      cloudSolrClient.add(doc);
    }
    cloudSolrClient.commit();
  }
  if (optimize) {
    cloudSolrClient.optimize();
  }
}
 

public static void index(MiniSolrCloudCluster cluster, String collection, SolrInputDocument doc, boolean doCommit) throws IOException, SolrServerException {
  CloudSolrClient client = createCloudClient(cluster, collection);
  try {
    client.add(doc);
    if (doCommit) {
      client.commit(true, true);
    } else {
      client.commit(true, false);
    }
  } finally {
    client.close();
  }
}
 

protected static long waitForClusterToSync(int numDocs, CloudSolrClient clusterSolrClient, String query) throws Exception {
  long start = System.nanoTime();
  QueryResponse response = null;
  while (System.nanoTime() - start <= TimeUnit.NANOSECONDS.convert(120, TimeUnit.SECONDS)) {
    clusterSolrClient.commit();
    response = clusterSolrClient.query(new SolrQuery(query));
    if (response.getResults().getNumFound() == numDocs) {
      break;
    }
    Thread.sleep(1000);
  }
  return response != null ? response.getResults().getNumFound() : 0;
}
 

protected void testCollectionCreateSearchDelete() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();
  String collectionName = "testkerberoscollection";

  // create collection
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1",
      NUM_SHARDS, REPLICATION_FACTOR);
  create.process(solrClient);
  // The metrics counter for wrong credentials here really just means  
  assertAuthMetricsMinimums(6, 3, 0, 3, 0, 0);

  SolrInputDocument doc = new SolrInputDocument();
  doc.setField("id", "1");
  solrClient.add(collectionName, doc);
  solrClient.commit(collectionName);
  assertAuthMetricsMinimums(10, 5, 0, 5, 0, 0);

  SolrQuery query = new SolrQuery();
  query.setQuery("*:*");
  QueryResponse rsp = solrClient.query(collectionName, query);
  assertEquals(1, rsp.getResults().getNumFound());

  CollectionAdminRequest.Delete deleteReq = CollectionAdminRequest.deleteCollection(collectionName);
  deleteReq.process(solrClient);
  AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName,
      solrClient.getZkStateReader(), true, 330);
  assertAuthMetricsMinimums(14, 8, 0, 6, 0, 0);
}
 

protected void deleteByQuery(String collection, String q) throws IOException, SolrServerException {
  CloudSolrClient client = createCloudClient(collection);
  try {
    client.deleteByQuery(q);
    client.commit(true, true);
  } finally {
    client.close();
  }
}
 

protected void deleteById(String collection, List<String> ids) throws IOException, SolrServerException {
  CloudSolrClient client = createCloudClient(collection);
  try {
    client.deleteById(ids);
    client.commit(true, true);
  } finally {
    client.close();
  }
}
 

protected void index(String collection, List<SolrInputDocument> docs) throws IOException, SolrServerException {
  CloudSolrClient client = createCloudClient(collection);
  try {
    client.add(docs);
    client.commit(true, true);
  } finally {
    client.close();
  }
}
 

protected void index(String collection, SolrInputDocument doc) throws IOException, SolrServerException {
  CloudSolrClient client = createCloudClient(collection);
  try {
    client.add(doc);
    client.commit(true, true);
  } finally {
    client.close();
  }
}
 

private String getDocFieldValue(CloudSolrClient clusterSolrClient, String query, String match, String field) throws Exception {
  TimeOut waitTimeOut = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  while (!waitTimeOut.hasTimedOut()) {
    clusterSolrClient.commit();
    QueryResponse response = clusterSolrClient.query(new SolrQuery(query));
    if (response.getResults().size() > 0 && match.equals(response.getResults().get(0).get(field))) {
      return (String) response.getResults().get(0).get(field);
    }
    Thread.sleep(1000);
  }
  return null;
}
 

@Test
public void testRepFactor1LeaderStartup() throws Exception {

  CloudSolrClient solrClient = cluster.getSolrClient();

  String collectionName = "testRepFactor1LeaderStartup";
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
      .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName());
  create.process(solrClient);

 cluster.waitForActiveCollection(collectionName, 1, 1);

  solrClient.setDefaultCollection(collectionName);

  String addListenerCommand = "{" +
      "'add-listener' : {'name':'newSearcherListener','event':'newSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "'add-listener' : {'name':'firstSearcherListener','event':'firstSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "}";

  ConfigRequest request = new ConfigRequest(addListenerCommand);
  solrClient.request(request);

  solrClient.add(new SolrInputDocument("id", "1"));
  solrClient.commit();

  AtomicInteger expectedDocs = new AtomicInteger(1);
  AtomicReference<String> failingCoreNodeName = new AtomicReference<>();
  CollectionStateWatcher stateWatcher = createActiveReplicaSearcherWatcher(expectedDocs, failingCoreNodeName);

  JettySolrRunner runner = cluster.getJettySolrRunner(0);
  runner.stop();
  
  cluster.waitForJettyToStop(runner);
  // check waitForState only after we are sure the node has shutdown and have forced an update to liveNodes
  // ie: workaround SOLR-13490
  cluster.getSolrClient().getZkStateReader().updateLiveNodes();
  waitForState("jetty count:" + cluster.getJettySolrRunners().size(), collectionName, clusterShape(1, 0));
  
  // restart
  sleepTime.set(1000);
  runner.start();
  cluster.waitForAllNodes(30);
  cluster.getSolrClient().getZkStateReader().registerCollectionStateWatcher(collectionName, stateWatcher);
  cluster.waitForActiveCollection(collectionName, 1, 1);
  assertNull("No replica should have been active without registering a searcher, found: " + failingCoreNodeName.get(), failingCoreNodeName.get());
  cluster.getSolrClient().getZkStateReader().removeCollectionStateWatcher(collectionName, stateWatcher);
}
 

@Test
public void test() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();
  solrClient.setDefaultCollection(COLLECTION_NAME);
  for (int i=0; i<10; i++)  {
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", String.valueOf(i));
    solrClient.add(doc);
  }
  solrClient.commit();

  // find the leader node
  Replica replica = solrClient.getZkStateReader().getLeaderRetry(COLLECTION_NAME, "shard1");
  JettySolrRunner replicaJetty = null;
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (JettySolrRunner jettySolrRunner : jettySolrRunners) {
    int port = jettySolrRunner.getBaseUrl().getPort();
    if (replica.getStr(BASE_URL_PROP).contains(":" + port))  {
      replicaJetty = jettySolrRunner;
      break;
    }
  }

  // kill the leader
  replicaJetty.stop();

  // add a replica (asynchronously)
  CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, "shard1");
  String asyncId = addReplica.processAsync(solrClient);

  // wait a bit
  Thread.sleep(1000);

  // bring the old leader node back up
  replicaJetty.start();

  // wait until everyone is active
  solrClient.waitForState(COLLECTION_NAME, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
      (n, c) -> DocCollection.isFullyActive(n, c, 1, 2));

  // now query each replica and check for consistency
  assertConsistentReplicas(solrClient, solrClient.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME).getSlice("shard1"));

  // sanity check that documents still exist
  QueryResponse response = solrClient.query(new SolrQuery("*:*"));
  assertEquals("Indexed documents not found", 10, response.getResults().getNumFound());
}
 

/**
 * Creates the cloud.
 */
private static void createCloud() {
  Path dataPath = Paths.get("src" + File.separator + "test" + File.separator
      + "resources" + File.separator + "data");
  String solrxml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML;
  JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").build();
  File cloudBase = Files.createTempDir();
  cloudBaseDir = cloudBase.toPath();
  // create subdirectories
  Path clusterDir = cloudBaseDir.resolve("cluster");
  Path logDir = cloudBaseDir.resolve("log");
  if (clusterDir.toFile().mkdir() && logDir.toFile().mkdir()) {
    // set log directory
    System.setProperty("solr.log.dir", logDir.toAbsolutePath().toString());
    try {
      cloudCluster = new MiniSolrCloudCluster(1, clusterDir, solrxml,
          jettyConfig);
      CloudSolrClient client = cloudCluster.getSolrClient();
      client.connect();
      createCloudCollection(COLLECTION_ALL_OPTIMIZED, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_ALL_MULTIPLE_SEGMENTS, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_PART1_OPTIMIZED, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_PART2_MULTIPLE_SEGMENTS, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_DISTRIBUTED, 1, 1,
          dataPath.resolve("conf"));

      // collection1
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(1));
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(2));
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(3));
      client.commit(COLLECTION_ALL_OPTIMIZED);
      client.optimize(COLLECTION_ALL_OPTIMIZED);
      // collection2
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(1));
      client.commit(COLLECTION_ALL_MULTIPLE_SEGMENTS);
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(2));
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(3));
      client.commit(COLLECTION_ALL_MULTIPLE_SEGMENTS);
      // collection3
      client.add(COLLECTION_PART1_OPTIMIZED, solrDocuments.get(1));
      client.commit(COLLECTION_PART1_OPTIMIZED);
      // collection4
      client.add(COLLECTION_PART2_MULTIPLE_SEGMENTS, solrDocuments.get(2));
      client.add(COLLECTION_PART2_MULTIPLE_SEGMENTS, solrDocuments.get(3));
      client.commit(COLLECTION_PART2_MULTIPLE_SEGMENTS);
    } catch (Exception e) {
      e.printStackTrace();
      log.error(e);
    }
  } else {
    log.error("couldn't create directories");
  }
}
 

@Test
public void test() throws Exception {
  setTestSuffix("testok");
  boolean isImplicit = random().nextBoolean();
  boolean doSplitShardOperation = !isImplicit && random().nextBoolean();
  replFactor = TestUtil.nextInt(random(), 1, 2);
  numTlogReplicas = TestUtil.nextInt(random(), 0, 1);
  numPullReplicas = TestUtil.nextInt(random(), 0, 1);
  int backupReplFactor = replFactor + numPullReplicas + numTlogReplicas;

  CollectionAdminRequest.Create create = isImplicit ?
      // NOTE: use shard list with same # of shards as NUM_SHARDS; we assume this later
      CollectionAdminRequest.createCollectionWithImplicitRouter(getCollectionName(), "conf1", "shard1,shard2", replFactor, numTlogReplicas, numPullReplicas) :
      CollectionAdminRequest.createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor, numTlogReplicas, numPullReplicas);

  if (random().nextBoolean()) {
    create.setMaxShardsPerNode(-1);
  } else if (doSplitShardOperation) {
    create.setMaxShardsPerNode((int) Math.ceil(NUM_SPLIT_SHARDS * backupReplFactor / (double) cluster.getJettySolrRunners().size()));
  } else if (NUM_SHARDS * (backupReplFactor) > cluster.getJettySolrRunners().size() || random().nextBoolean()) {
    create.setMaxShardsPerNode((int) Math.ceil(NUM_SHARDS * backupReplFactor / (double) cluster.getJettySolrRunners().size()));//just to assert it survives the restoration
  }

  if (random().nextBoolean()) {
    create.setAutoAddReplicas(true);//just to assert it survives the restoration
  }
  Properties coreProps = new Properties();
  coreProps.put("customKey", "customValue");//just to assert it survives the restoration
  create.setProperties(coreProps);
  if (isImplicit) { //implicit router
    create.setRouterField("shard_s");
  } else {//composite id router
    if (random().nextBoolean()) {
      create.setRouterField("shard_s");
    }
  }

  CloudSolrClient solrClient = cluster.getSolrClient();
  create.process(solrClient);

  indexDocs(getCollectionName(), false);

  if (doSplitShardOperation) {
    // shard split the first shard
    int prevActiveSliceCount = getActiveSliceCount(getCollectionName());
    CollectionAdminRequest.SplitShard splitShard = CollectionAdminRequest.splitShard(getCollectionName());
    splitShard.setShardName("shard1");
    splitShard.process(solrClient);
    // wait until we see one more active slice...
    for (int i = 0; getActiveSliceCount(getCollectionName()) != prevActiveSliceCount + 1; i++) {
      assertTrue(i < 30);
      Thread.sleep(500);
    }
    // issue a hard commit.  Split shard does a soft commit which isn't good enough for the backup/snapshooter to see
    solrClient.commit(getCollectionName());
  }

  testBackupAndRestore(getCollectionName(), backupReplFactor);
  testConfigBackupOnly("conf1", getCollectionName());
  testInvalidPath(getCollectionName());
}
 

@Test
public void testJmxReporter() throws Exception {
  CollectionAdminRequest.reloadCollection(COLLECTION).processAndWait(cluster.getSolrClient(), 60);
  CloudSolrClient solrClient = cluster.getSolrClient();
  // index some docs
  for (int i = 0; i < 100; i++) {
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", "id-" + i);
    solrClient.add(COLLECTION, doc);
  }
  solrClient.commit(COLLECTION);
  // make sure searcher is present
  solrClient.query(COLLECTION, params(CommonParams.Q, "*:*"));

  for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
    SolrMetricManager manager = runner.getCoreContainer().getMetricManager();
    for (String registry : manager.registryNames()) {
      Map<String, SolrMetricReporter> reporters = manager.getReporters(registry);
      long jmxReporters = reporters.entrySet().stream().filter(e -> e.getValue() instanceof SolrJmxReporter).count();
      reporters.forEach((k, v) -> {
        if (!(v instanceof SolrJmxReporter)) {
          return;
        }
        if (!((SolrJmxReporter)v).getDomain().startsWith("solr.core")) {
          return;
        }
        if (!((SolrJmxReporter)v).isActive()) {
          return;
        }
        QueryExp exp = Query.eq(Query.attr(JmxMetricsReporter.INSTANCE_TAG), Query.value(Integer.toHexString(v.hashCode())));
        Set<ObjectInstance> beans = mBeanServer.queryMBeans(null, exp);
        if (((SolrJmxReporter) v).isStarted() && beans.isEmpty() && jmxReporters < 2) {
          if (log.isInfoEnabled()) {
            log.info("DocCollection: {}", getCollectionState(COLLECTION));
          }
          fail("JMX reporter " + k + " for registry " + registry + " failed to register any beans!");
        } else {
          Set<String> categories = new HashSet<>();
          beans.forEach(bean -> {
            String cat = bean.getObjectName().getKeyProperty("category");
            if (cat != null) {
              categories.add(cat);
            }
          });
          log.info("Registered categories: {}", categories);
          assertTrue("Too few categories: " + categories, categories.size() > 5);
        }
      });
    }
  }
}