Java Code Examples for org.apache.solr.client.solrj.embedded.JettySolrRunner#stop()

public void testOutOfOrderDBQWithInPlaceUpdates() throws Exception {
  createAndWaitForCollection(1,0,2,0);
  assertFalse(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").indexed());
  assertFalse(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").stored());
  assertTrue(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").hasDocValues());
  List<UpdateRequest> updates = new ArrayList<>();
  updates.add(simulatedUpdateRequest(null, "id", 1, "title_s", "title0_new", "inplace_updatable_int", 5, "_version_", 1L)); // full update
  updates.add(simulatedDBQ("inplace_updatable_int:5", 3L));
  updates.add(simulatedUpdateRequest(1L, "id", 1, "inplace_updatable_int", 6, "_version_", 2L));
  for (JettySolrRunner solrRunner: getSolrRunner(false)) {
    try (SolrClient client = solrRunner.newClient()) {
      for (UpdateRequest up : updates) {
        up.process(client, collectionName);
      }
    }
  }
  JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
  oldLeaderJetty.stop();
  waitForState("Replica not removed", collectionName, activeReplicaCount(0, 1, 0));
  waitForLeaderChange(oldLeaderJetty, "shard1");
  oldLeaderJetty.start();
  waitForState("Replica not added", collectionName, activeReplicaCount(0, 2, 0));
  checkRTG(1,1, cluster.getJettySolrRunners());
  SolrDocument doc = cluster.getSolrClient().getById(collectionName,"1");
  assertNotNull(doc.get("title_s"));
}
 

@Test
public void testHealthCheckV2Api() throws Exception {
  V2Response res = new V2Request.Builder("/node/health").build().process(cluster.getSolrClient());
  assertEquals(0, res.getStatus());
  assertEquals(CommonParams.OK, res.getResponse().get(CommonParams.STATUS));

  // add a new node for the purpose of negative testing
  JettySolrRunner newJetty = cluster.startJettySolrRunner();
  try (HttpSolrClient httpSolrClient = getHttpSolrClient(newJetty.getBaseUrl().toString())) {

    // postive check that our (new) "healthy" node works with direct http client
    assertEquals(CommonParams.OK, new V2Request.Builder("/node/health").build().process(httpSolrClient).
        getResponse().get(CommonParams.STATUS));

    // now "break" our (new) node
    newJetty.getCoreContainer().getZkController().getZkClient().close();

    // negative check of our (new) "broken" node that we deliberately put into an unhealth state
    BaseHttpSolrClient.RemoteSolrException e = expectThrows(BaseHttpSolrClient.RemoteSolrException.class, () ->
    {
      new V2Request.Builder("/node/health").build().process(httpSolrClient);
    });
    assertTrue(e.getMessage(), e.getMessage().contains("Host Unavailable"));
    assertEquals(SolrException.ErrorCode.SERVICE_UNAVAILABLE.code, e.code());
  } finally {
    newJetty.stop();
  }
}
 

@Override
protected void destroyServers() throws Exception {
  for (JettySolrRunner runner : jettys) {
    try {
      runner.stop();
    } catch (Exception e) {
      log.error("", e);
    }
  }

  jettys.clear();
}
 

public void testKillPullReplica() throws Exception {
    CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 1)
      .setMaxShardsPerNode(100)
      .process(cluster.getSolrClient());
//    cluster.getSolrClient().getZkStateReader().registerCore(collectionName); //TODO: Is this needed?
    waitForState("Expected collection to be created with 1 shard and 2 replicas", collectionName, clusterShape(1, 2));
    DocCollection docCollection = assertNumberOfReplicas(1, 0, 1, false, true);
    assertEquals(1, docCollection.getSlices().size());

    waitForNumDocsInAllActiveReplicas(0);
    cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
    cluster.getSolrClient().commit(collectionName);
    waitForNumDocsInAllActiveReplicas(1);

    JettySolrRunner pullReplicaJetty = cluster.getReplicaJetty(docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0));
    pullReplicaJetty.stop();
    waitForState("Replica not removed", collectionName, activeReplicaCount(1, 0, 0));
    // Also wait for the replica to be placed in state="down"
    waitForState("Didn't update state", collectionName, clusterStateReflectsActiveAndDownReplicas());

    cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "bar"));
    cluster.getSolrClient().commit(collectionName);
    waitForNumDocsInAllActiveReplicas(2);

    pullReplicaJetty.start();
    waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 1));
    waitForNumDocsInAllActiveReplicas(2);
  }
 

@Test
public void testDesignatedOverseerRestarts() throws Exception {
  logOverseerState();
  // Remove the OVERSEER role, in case it was already assigned by another test in this suite
  for (String node: OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient())) {
    CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
  }
  String overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
  int counter = 0;
  while (overseer1 == null && counter < 10) {
    overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
    Thread.sleep(1000);
  }

  // Setting overseer role to the current overseer
  CollectionAdminRequest.addRole(overseer1, "overseer").process(cluster.getSolrClient());
  waitForNewOverseer(15, overseer1, false);
  JettySolrRunner leaderJetty = getOverseerJetty();

  List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());
  nodes.remove(overseer1); // remove the designated overseer

  logOverseerState();
  // kill the current overseer, and check that the next node in the election queue assumes leadership
  leaderJetty.stop();
  log.info("Killing designated overseer: {}", overseer1);

  // after 5 seconds, bring back dead designated overseer and assert that it assumes leadership "right away",
  // i.e. without any other node assuming leadership before this node becomes leader.
  Thread.sleep(5);
  logOverseerState();
  log.info("Starting back the prioritized overseer..");
  leaderJetty.start();
  waitForNewOverseer(15, overseer1, true); // assert that there is just a single leadership transition
}
 

/**
 * Stop a Solr instance
 * @param index the index of node in collection returned by {@link #getJettySolrRunners()}
 * @return the shut down node
 */
public JettySolrRunner stopJettySolrRunner(int index) throws Exception {
  JettySolrRunner jetty = jettys.get(index);
  jetty.stop();
  jettys.remove(index);
  return jetty;
}
 

public void testWaitForStateAfterShutDown() throws Exception {
  final String col_name = "test_col";
  final MiniSolrCloudCluster cluster = new MiniSolrCloudCluster
    (1, createTempDir(), buildJettyConfig("/solr"));
  try {
    log.info("Create our collection");
    CollectionAdminRequest.createCollection(col_name, "_default", 1, 1).process(cluster.getSolrClient());
    
    log.info("Sanity check that our collection has come online");
    cluster.getSolrClient().waitForState(col_name, 30, TimeUnit.SECONDS, clusterShape(1, 1));
                                         
    log.info("Shutdown 1 node");
    final JettySolrRunner nodeToStop = cluster.getJettySolrRunner(0);
    nodeToStop.stop();
    log.info("Wait to confirm our node is fully shutdown");
    cluster.waitForJettyToStop(nodeToStop);

    // now that we're confident that node has stoped, check if a waitForState
    // call will detect the missing replica -- shouldn't need long wait times (we know it's down)...
    log.info("Now check if waitForState will recognize we already have the exepcted state");
    cluster.getSolrClient().waitForState(col_name, 500, TimeUnit.MILLISECONDS, clusterShape(1, 0));
                                         
    
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * Restart a server.
 */
protected void restartServer(CloudJettyRunner server) throws Exception {
  // it seems we need to set the collection property to have the jetty properly restarted
  System.setProperty("collection", server.collection);
  JettySolrRunner jetty = server.jetty;
  jetty.stop();
  jetty.start();
  System.clearProperty("collection");
  waitForRecoveriesToFinish(server.collection, true);
  updateMappingsFromZk(server.collection); // must update the mapping as the core node name might have changed
}
 

private void testDoRecoveryOnRestart() throws Exception {
  String testCollectionName = "collDoRecoveryOnRestart";
  try {
    // Inject pausing in recovery op, hence the replica won't be able to finish recovery

    TestInjection.prepRecoveryOpPauseForever = "true:100";
    
    createCollection(testCollectionName, "conf1", 1, 2, 1);
    cloudClient.setDefaultCollection(testCollectionName);

    sendDoc(1, 2);

    JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(getShardLeader(testCollectionName, "shard1", 1000)));
    List<Replica> notLeaders =
        ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, maxWaitSecsToSeeAllActive);
    assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 1);

    SocketProxy proxy0 = getProxyForReplica(notLeaders.get(0));
    SocketProxy leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));

    proxy0.close();
    leaderProxy.close();

    // indexing during a partition
    int achievedRf = sendDoc(2, 1, leaderJetty);
    assertEquals("Unexpected achieved replication factor", 1, achievedRf);
    try (ZkShardTerms zkShardTerms = new ZkShardTerms(testCollectionName, "shard1", cloudClient.getZkStateReader().getZkClient())) {
      assertFalse(zkShardTerms.canBecomeLeader(notLeaders.get(0).getName()));
    }
    waitForState(testCollectionName, notLeaders.get(0).getName(), DOWN, 10000);

    // heal partition
    proxy0.reopen();
    leaderProxy.reopen();

    waitForState(testCollectionName, notLeaders.get(0).getName(), RECOVERING, 10000);

    System.clearProperty("solrcloud.skip.autorecovery");
    JettySolrRunner notLeaderJetty = getJettyOnPort(getReplicaPort(notLeaders.get(0)));
    String notLeaderNodeName = notLeaderJetty.getNodeName();
    notLeaderJetty.stop();
    
    cloudClient.getZkStateReader().waitForLiveNodes(15, TimeUnit.SECONDS, SolrCloudTestCase.missingLiveNode(notLeaderNodeName));

    notLeaderJetty.start();
    ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, 130);
    assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 2);
  } finally {
    TestInjection.prepRecoveryOpPauseForever = null;
    TestInjection.notifyPauseForeverDone();
  }

  // try to clean up
  attemptCollectionDelete(cloudClient, testCollectionName);
}
 

/**
 * Test that basic autoAddReplicaLogic logic is <b>not</b> used if the cluster prop for it is disabled 
 * (even if sys prop is set after collection is created)
 */
@Test
public void testClusterPropOverridesCollecitonProp() throws Exception {
  final String COLLECTION = "test_clusterprop";
  final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
  final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
  final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);

  if (log.isInfoEnabled()) {
    log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
        jetty1.getNodeName(), jetty1.getLocalPort(),
        jetty2.getNodeName(), jetty2.getLocalPort());
  }
           
  CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
    .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
    .setAutoAddReplicas(true)
    .setMaxShardsPerNode(2)
    .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(COLLECTION, 2, 4);

  // check cluster property is considered
  disableAutoAddReplicasInCluster();

  JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
  String lostNodeName = lostJetty.getNodeName();
  List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);

  if (log.isInfoEnabled()) {
    log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.stop();
  
  cluster.waitForJettyToStop(lostJetty);
  
  waitForNodeLeave(lostNodeName);
  
  waitForState(COLLECTION + "=(2,2)", COLLECTION,
               clusterShape(2, 2), 90, TimeUnit.SECONDS);
               

  if (log.isInfoEnabled()) {
    log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.start();
  
  waitForNodeLive(lostJetty);
  
  assertTrue("Timeout waiting for all live and active",
             ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
  
  waitForState(COLLECTION + "=(2,4) w/o down replicas",
               COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);

}
 

/**
 * Test that basic autoAddReplicaLogic kicks in when a node is lost 
 */
@Test
public void testSimple() throws Exception {
  final String COLLECTION = "test_simple";
  final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
  final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
  final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);
  if (log.isInfoEnabled()) {
    log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
        jetty1.getNodeName(), jetty1.getLocalPort(),
        jetty2.getNodeName(), jetty2.getLocalPort());
  }
           
  CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
    .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
    .setAutoAddReplicas(true)
    .setMaxShardsPerNode(2)
    .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(COLLECTION, 2, 4);
  
  // start the tests
  JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
  String lostNodeName = lostJetty.getNodeName();
  List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);
  if (log.isInfoEnabled()) {
    log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.stop();
  
  cluster.waitForJettyToStop(lostJetty);
  waitForNodeLeave(lostNodeName);
  
  waitForState(COLLECTION + "=(2,4) w/o down replicas",
               COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
               
  checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);

  if (log.isInfoEnabled()) {
    log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.start();
  
  waitForNodeLive(lostJetty);
  
  assertTrue("Timeout waiting for all live and active",
             ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));

}
 

@Test
public void testMostInSyncReplicasCanWinElection() throws Exception {
  final String collectionName = "collection1";
  CollectionAdminRequest.createCollection(collectionName, 1, 3)
      .setCreateNodeSet("")
      .process(cluster.getSolrClient());
  CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
      .setNode(cluster.getJettySolrRunner(0).getNodeName())
      .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 1, 1);
  
  waitForState("Timeout waiting for shard leader", collectionName, clusterShape(1, 1));
  Replica leader = getCollectionState(collectionName).getSlice("shard1").getLeader();

  // this replica will ahead of election queue
  CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
      .setNode(cluster.getJettySolrRunner(1).getNodeName())
      .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 1, 2);
  
  waitForState("Timeout waiting for 1x2 collection", collectionName, clusterShape(1, 2));
  Replica replica1 = getCollectionState(collectionName).getSlice("shard1")
      .getReplicas(replica -> replica.getNodeName().equals(cluster.getJettySolrRunner(1).getNodeName())).get(0);

  CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
      .setNode(cluster.getJettySolrRunner(2).getNodeName())
      .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 1, 3);
  
  waitForState("Timeout waiting for 1x3 collection", collectionName, clusterShape(1, 3));
  Replica replica2 = getCollectionState(collectionName).getSlice("shard1")
      .getReplicas(replica -> replica.getNodeName().equals(cluster.getJettySolrRunner(2).getNodeName())).get(0);

  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1"));
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2"));
  cluster.getSolrClient().commit(collectionName);

  // replica in node1 won't be able to do recovery
  proxies.get(cluster.getJettySolrRunner(0)).close();
  // leader won't be able to send request to replica in node1
  proxies.get(cluster.getJettySolrRunner(1)).close();

  addDoc(collectionName, 3, cluster.getJettySolrRunner(0));

  try (ZkShardTerms zkShardTerms = new ZkShardTerms(collectionName, "shard1", cluster.getZkClient())) {
    assertEquals(3, zkShardTerms.getTerms().size());
    assertEquals(zkShardTerms.getHighestTerm(), zkShardTerms.getTerm(leader.getName()));
    assertEquals(zkShardTerms.getHighestTerm(), zkShardTerms.getTerm(replica2.getName()));
    assertTrue(zkShardTerms.getHighestTerm() > zkShardTerms.getTerm(replica1.getName()));
  }

  proxies.get(cluster.getJettySolrRunner(2)).close();
  addDoc(collectionName, 4, cluster.getJettySolrRunner(0));

  try (ZkShardTerms zkShardTerms = new ZkShardTerms(collectionName, "shard1", cluster.getZkClient())) {
    assertEquals(3, zkShardTerms.getTerms().size());
    assertEquals(zkShardTerms.getHighestTerm(), zkShardTerms.getTerm(leader.getName()));
    assertTrue(zkShardTerms.getHighestTerm() > zkShardTerms.getTerm(replica2.getName()));
    assertTrue(zkShardTerms.getHighestTerm() > zkShardTerms.getTerm(replica1.getName()));
    assertTrue(zkShardTerms.getTerm(replica2.getName()) > zkShardTerms.getTerm(replica1.getName()));
  }

  proxies.get(cluster.getJettySolrRunner(1)).reopen();
  proxies.get(cluster.getJettySolrRunner(2)).reopen();
  
  
  JettySolrRunner j = cluster.getJettySolrRunner(0);
  j.stop();
  cluster.waitForJettyToStop(j);

  try {
    // even replica2 joined election at the end of the queue, but it is the one with highest term
    waitForState("Timeout waiting for new leader", collectionName, (liveNodes, collectionState) -> {
      Replica newLeader = collectionState.getSlice("shard1").getLeader();
      if (newLeader == null) {
        return false;
      }
      return newLeader.getName().equals(replica2.getName());
    });
  } catch (Exception e) {
    List<String> children = zkClient().getChildren("/collections/"+collectionName+"/leader_elect/shard1/election",
        null, true);
    log.info("{} election nodes:{}", collectionName, children);
    throw e;
  }
  cluster.getJettySolrRunner(0).start();
  proxies.get(cluster.getJettySolrRunner(0)).reopen();

  waitForState("Timeout waiting for 1x3 collection", collectionName, clusterShape(1, 3));
  assertDocsExistInAllReplicas(Arrays.asList(leader, replica1), collectionName, 1, 3);
  CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
}
 

@Test
public void testRepFactor1LeaderStartup() throws Exception {

  CloudSolrClient solrClient = cluster.getSolrClient();

  String collectionName = "testRepFactor1LeaderStartup";
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
      .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName());
  create.process(solrClient);

 cluster.waitForActiveCollection(collectionName, 1, 1);

  solrClient.setDefaultCollection(collectionName);

  String addListenerCommand = "{" +
      "'add-listener' : {'name':'newSearcherListener','event':'newSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "'add-listener' : {'name':'firstSearcherListener','event':'firstSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "}";

  ConfigRequest request = new ConfigRequest(addListenerCommand);
  solrClient.request(request);

  solrClient.add(new SolrInputDocument("id", "1"));
  solrClient.commit();

  AtomicInteger expectedDocs = new AtomicInteger(1);
  AtomicReference<String> failingCoreNodeName = new AtomicReference<>();
  CollectionStateWatcher stateWatcher = createActiveReplicaSearcherWatcher(expectedDocs, failingCoreNodeName);

  JettySolrRunner runner = cluster.getJettySolrRunner(0);
  runner.stop();
  
  cluster.waitForJettyToStop(runner);
  // check waitForState only after we are sure the node has shutdown and have forced an update to liveNodes
  // ie: workaround SOLR-13490
  cluster.getSolrClient().getZkStateReader().updateLiveNodes();
  waitForState("jetty count:" + cluster.getJettySolrRunners().size(), collectionName, clusterShape(1, 0));
  
  // restart
  sleepTime.set(1000);
  runner.start();
  cluster.waitForAllNodes(30);
  cluster.getSolrClient().getZkStateReader().registerCollectionStateWatcher(collectionName, stateWatcher);
  cluster.waitForActiveCollection(collectionName, 1, 1);
  assertNull("No replica should have been active without registering a searcher, found: " + failingCoreNodeName.get(), failingCoreNodeName.get());
  cluster.getSolrClient().getZkStateReader().removeCollectionStateWatcher(collectionName, stateWatcher);
}
 

private void doReplaceLeader(boolean removeReplica) throws Exception {
  DocCollection docCollection = createAndWaitForCollection(1, 0, 2, 0);

  // Add a document and commit
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
  cluster.getSolrClient().commit(collectionName);
  Slice s = docCollection.getSlices().iterator().next();
  try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
    assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
  }

  waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);

  // Delete leader replica from shard1
  JettySolrRunner leaderJetty = null;
  if (removeReplica) {
    CollectionAdminRequest.deleteReplica(
        collectionName,
        "shard1",
        s.getLeader().getName())
    .process(cluster.getSolrClient());
  } else {
    leaderJetty = cluster.getReplicaJetty(s.getLeader());
    leaderJetty.stop();
    waitForState("Leader replica not removed", collectionName, clusterShape(1, 1));
    // Wait for cluster state to be updated
    waitForState("Replica state not updated in cluster state",
        collectionName, clusterStateReflectsActiveAndDownReplicas());
  }
  docCollection = assertNumberOfReplicas(0, 1, 0, true, true);

  // Wait until a new leader is elected
  waitForLeaderChange(leaderJetty, "shard1");
  
  // There is a new leader, I should be able to add and commit
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"));
  cluster.getSolrClient().commit(collectionName);

  // Queries should still work
  waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
  // Start back the node
  if (removeReplica) {
    addReplicaWithRetries();
    
  } else {
    leaderJetty.start();
  }
  waitForState("Expected collection to be 1x2", collectionName, clusterShape(1, 2));
  // added replica should replicate from the leader
  waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
}
 

@Test
public void test() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();
  solrClient.setDefaultCollection(COLLECTION_NAME);
  for (int i=0; i<10; i++)  {
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", String.valueOf(i));
    solrClient.add(doc);
  }
  solrClient.commit();

  // find the leader node
  Replica replica = solrClient.getZkStateReader().getLeaderRetry(COLLECTION_NAME, "shard1");
  JettySolrRunner replicaJetty = null;
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (JettySolrRunner jettySolrRunner : jettySolrRunners) {
    int port = jettySolrRunner.getBaseUrl().getPort();
    if (replica.getStr(BASE_URL_PROP).contains(":" + port))  {
      replicaJetty = jettySolrRunner;
      break;
    }
  }

  // kill the leader
  replicaJetty.stop();

  // add a replica (asynchronously)
  CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, "shard1");
  String asyncId = addReplica.processAsync(solrClient);

  // wait a bit
  Thread.sleep(1000);

  // bring the old leader node back up
  replicaJetty.start();

  // wait until everyone is active
  solrClient.waitForState(COLLECTION_NAME, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
      (n, c) -> DocCollection.isFullyActive(n, c, 1, 2));

  // now query each replica and check for consistency
  assertConsistentReplicas(solrClient, solrClient.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME).getSlice("shard1"));

  // sanity check that documents still exist
  QueryResponse response = solrClient.query(new SolrQuery("*:*"));
  assertEquals("Indexed documents not found", 10, response.getResults().getNumFound());
}
 

@Test
public void testOverseerRole() throws Exception {

  logOverseerState();
  List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());
  // Remove the OVERSEER role, in case it was already assigned by another test in this suite
  for (String node: nodes) {
    CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
  }
  String overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
  nodes.remove(overseer1);

  Collections.shuffle(nodes, random());
  String overseer2 = nodes.get(0);
  log.info("### Setting overseer designate {}", overseer2);

  CollectionAdminRequest.addRole(overseer2, "overseer").process(cluster.getSolrClient());

  waitForNewOverseer(15, overseer2, false);

  //add another node as overseer
  nodes.remove(overseer2);
  Collections.shuffle(nodes, random());

  String overseer3 = nodes.get(0);
  log.info("### Adding another overseer designate {}", overseer3);
  CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient());

  // kill the current overseer, and check that the new designate becomes the new overseer
  JettySolrRunner leaderJetty = getOverseerJetty();
  logOverseerState();

  leaderJetty.stop();
  waitForNewOverseer(10, overseer3, false);

  // add another node as overseer
  nodes.remove(overseer3);
  Collections.shuffle(nodes, random());
  String overseer4 = nodes.get(0);
  log.info("### Adding last overseer designate {}", overseer4);
  CollectionAdminRequest.addRole(overseer4, "overseer").process(cluster.getSolrClient());
  logOverseerState();

  // remove the overseer role from the current overseer
  CollectionAdminRequest.removeRole(overseer3, "overseer").process(cluster.getSolrClient());
  waitForNewOverseer(15, overseer4, false);

  // Add it back again - we now have two delegates, 4 and 3
  CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient());

  // explicitly tell the overseer to quit
  String leaderId = OverseerCollectionConfigSetProcessor.getLeaderId(zkClient());
  String leader = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
  log.info("### Sending QUIT to overseer {}", leader);
  getOverseerJetty().getCoreContainer().getZkController().getOverseer().getStateUpdateQueue()
      .offer(Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(),
          "id", leaderId)));

  waitForNewOverseer(15, s -> Objects.equals(leader, s) == false, false);

  Thread.sleep(1000);
  
  logOverseerState();
  assertTrue("The old leader should have rejoined election",
      OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()).contains(leader));

  leaderJetty.start(); // starting this back, just for good measure
}
 

@Test
//28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 16-Apr-2018
public void testListenerAcceptance() throws Exception {

  CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
  Map<String, Object> props = createTriggerProps(0);

  try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger")) {
    trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
    trigger.init();
    trigger.setProcessor(noFirstRunProcessor);

    JettySolrRunner newNode = cluster.startJettySolrRunner();

    cluster.waitForAllNodes(30);
    
    trigger.run(); // starts tracking live nodes
    
    // stop the newly created node
    newNode.stop();
    cluster.waitForJettyToStop(newNode);

    AtomicInteger callCount = new AtomicInteger(0);
    AtomicBoolean fired = new AtomicBoolean(false);

    trigger.setProcessor(event -> {
      if (callCount.incrementAndGet() < 2) {
        return false;
      } else  {
        fired.compareAndSet(false, true);
        return true;
      }
    });

    Thread.sleep(1000);
    
    trigger.run(); // first run should detect the lost node and fire immediately but listener isn't ready
    
    TimeOut timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME);
    timeout.waitFor("Timeout waiting for callCount to hit at least 1", () -> callCount.get() >= 1);
    assertEquals(1, callCount.get());
    assertFalse(fired.get());
    trigger.run(); // second run should again fire
    timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME);
    timeout.waitFor("Timeout waiting for callCount to hit at least 2", () -> callCount.get() >= 2);
    assertEquals(2, callCount.get());
    assertTrue(fired.get());
    trigger.run(); // should not fire
    assertEquals(2, callCount.get());
  }
}
 

@Test
//commented 2-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018
public void test() throws Exception {

  final String collection = "recoverytest";

  CollectionAdminRequest.createCollection(collection, "conf", 1, 2)
      .setMaxShardsPerNode(1)
      .process(cluster.getSolrClient());
  waitForState("Expected a collection with one shard and two replicas", collection, clusterShape(1, 2));
  cluster.getSolrClient().setDefaultCollection(collection);

  // start a couple indexing threads
  
  int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000};
  int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000};
  
  int maxDoc;
  if (!TEST_NIGHTLY) {
    maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
  } else {
    maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)];
  }
  log.info("Indexing {} documents", maxDoc);
  
  final StoppableIndexingThread indexThread
    = new StoppableIndexingThread(null, cluster.getSolrClient(), "1", true, maxDoc, 1, true);
  threads.add(indexThread);
  indexThread.start();
  
  final StoppableIndexingThread indexThread2
    = new StoppableIndexingThread(null, cluster.getSolrClient(), "2", true, maxDoc, 1, true);
  threads.add(indexThread2);
  indexThread2.start();

  // give some time to index...
  int[] waitTimes = new int[] {200, 2000, 3000};
  Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
   
  // bring shard replica down
  DocCollection state = getCollectionState(collection);
  Replica leader = state.getLeader("shard1");
  Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> leader != r);

  JettySolrRunner jetty = cluster.getReplicaJetty(replica);
  jetty.stop();
  
  // wait a moment - lets allow some docs to be indexed so replication time is non 0
  Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
  
  // bring shard replica up
  jetty.start();
  
  // make sure replication can start
  Thread.sleep(3000);

  // stop indexing threads
  indexThread.safeStop();
  indexThread2.safeStop();
  
  indexThread.join();
  indexThread2.join();

  new UpdateRequest()
      .commit(cluster.getSolrClient(), collection);

  cluster.getSolrClient().waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2));

  // test that leader and replica have same doc count
  state = getCollectionState(collection);
  assertShardConsistency(state.getSlice("shard1"), true);

}
 

/**
 * Test that we can modify a collection after creation to add autoAddReplicas.
 */
@Test
public void testAddCollectionPropAfterCreation() throws Exception {
  final String COLLECTION = "test_addprop";
  final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
  final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
  final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);

  if (log.isInfoEnabled()) {
    log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
        jetty1.getNodeName(), jetty1.getLocalPort(),
        jetty2.getNodeName(), jetty2.getLocalPort());
  }
           
  CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
    .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
    .setAutoAddReplicas(false) // NOTE: false
    .setMaxShardsPerNode(2)
    .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(COLLECTION, 2, 4);
  
  log.info("Modifying {} to use autoAddReplicas", COLLECTION);
  new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) {
    @Override
    public SolrParams getParams() {
      ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
      params.set("collection", COLLECTION);
      params.set("autoAddReplicas", true);
      return params;
    }
  }.process(cluster.getSolrClient());

  JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
  String lostNodeName = lostJetty.getNodeName();
  List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);

  if (log.isInfoEnabled()) {
    log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.stop();
  
  cluster.waitForJettyToStop(lostJetty);
  
  waitForNodeLeave(lostNodeName);

  waitForState(COLLECTION + "=(2,4) w/o down replicas",
               COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
  checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);

  if (log.isInfoEnabled()) {
    log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.start();
  
  waitForNodeLive(lostJetty);
  
  assertTrue("Timeout waiting for all live and active",
             ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
}
 

/**
 * Stop the given Solr instance. It will be removed from the cluster's list of running instances.
 * @param jetty a {@link JettySolrRunner} to be stopped
 * @return the same {@link JettySolrRunner} instance provided to this method
 * @throws Exception on error
 */
public JettySolrRunner stopJettySolrRunner(JettySolrRunner jetty) throws Exception {
  jetty.stop();
  jettys.remove(jetty);
  return jetty;
}