Java Code Examples for org.apache.solr.client.solrj.embedded.JettySolrRunner#start()

public void startJetty() throws Exception {

      Properties props = new Properties();
      props.setProperty("solrconfig", "bad_solrconfig.xml");
      props.setProperty("solr.data.dir", getDataDir());

      JettyConfig jettyConfig = JettyConfig.builder(buildJettyConfig("/solr")).setPort(port).build();

      jetty = new JettySolrRunner(getHomeDir(), props, jettyConfig);
      jetty.start();
      int newPort = jetty.getLocalPort();
      if (port != 0 && newPort != port) {
        fail("TESTING FAILURE: could not grab requested port.");
      }
      this.port = newPort;
//      System.out.println("waiting.........");
//      Thread.sleep(5000);
    }
 

public static JettySolrRunner createAndStartJetty(String solrHome, Properties nodeProperties, JettyConfig jettyConfig) throws Exception {

    initCore(null, null, solrHome);

    Path coresDir = createTempDir().resolve("cores");

    Properties props = new Properties();
    props.setProperty("name", DEFAULT_TEST_CORENAME);
    props.setProperty("configSet", "collection1");
    props.setProperty("config", "${solrconfig:solrconfig.xml}");
    props.setProperty("schema", "${schema:schema.xml}");

    writeCoreProperties(coresDir.resolve("core"), props, "RestTestBase");

    Properties nodeProps = new Properties(nodeProperties);
    nodeProps.setProperty("coreRootDirectory", coresDir.toString());
    nodeProps.setProperty("configSetBaseDir", solrHome);

    jetty = new JettySolrRunner(solrHome, nodeProps, jettyConfig);
    jetty.start();
    port = jetty.getLocalPort();
    log.info("Jetty Assigned Port#{}", port);
    return jetty;
  }
 

public JettySolrRunner createJetty(String dataDir, String ulogDir, String shardList,
    String solrConfigOverride) throws Exception {

  JettyConfig jettyconfig = JettyConfig.builder()
      .setContext(context)
      .stopAtShutdown(false)
      .withServlets(getExtraServlets())
      .withFilters(getExtraRequestFilters())
      .withSSLConfig(sslConfig.buildServerSSLConfig())
      .build();

  Properties props = new Properties();
  props.setProperty("solr.data.dir", getDataDir(dataDir));
  props.setProperty("shards", shardList);
  props.setProperty("solr.ulog.dir", ulogDir);
  props.setProperty("solrconfig", solrConfigOverride);
  
  JettySolrRunner jetty = new JettySolrRunner(getSolrHome(), props, jettyconfig);

  jetty.start();

  return jetty;
}
 

@Override
public void tearDown() throws Exception {
  for (JettySolrRunner jetty:cluster.getJettySolrRunners()) {
    if (!jetty.isRunning()) {
      log.warn("Jetty {} not running, probably some bad test. Starting it", jetty.getLocalPort());
      jetty.start();
    }
  }
  if (cluster.getSolrClient().getZkStateReader().getClusterState().getCollectionOrNull(collectionName) != null) {
    log.info("tearDown deleting collection");
    CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
    log.info("Collection deleted");
    waitForDeletion(collectionName);
  }
  super.tearDown();
}
 

@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
public void testLeaderRecovery() throws Exception {
  System.setProperty("CoreInitFailedAction", "fromleader");

  // Simulate failure by truncating the segment_* files
  for (File segment : getSegmentFiles(replica)) {
    truncate(segment);
  }

  // Might not need a sledge-hammer to reload the core
  JettySolrRunner jetty = cluster.getReplicaJetty(replica);
  jetty.stop();
  jetty.start();

  waitForState("Expected a collection with one shard and two replicas", collection, clusterShape(1, 2));
  
  QueryResponse resp = cluster.getSolrClient().query(collection, new SolrQuery("*:*"));
  assertEquals(10, resp.getResults().getNumFound());
}
 

@Ignore
  public void testKillTlogReplica() throws Exception {
    DocCollection docCollection = createAndWaitForCollection(1, 0, 2, 0);

    waitForNumDocsInAllActiveReplicas(0);
    cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
    cluster.getSolrClient().commit(collectionName);
    waitForNumDocsInAllActiveReplicas(1);

    JettySolrRunner pullReplicaJetty = cluster.getReplicaJetty(docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.TLOG)).get(0));
    pullReplicaJetty.stop();
    waitForState("Replica not removed", collectionName, activeReplicaCount(0, 1, 0));
    waitForLeaderChange(pullReplicaJetty, "shard1");
//    // Also wait for the replica to be placed in state="down"
//    waitForState("Didn't update state", collectionName, clusterStateReflectsActiveAndDownReplicas());

    cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "bar"));
    cluster.getSolrClient().commit(collectionName);
    waitForNumDocsInAllActiveReplicas(2);

    pullReplicaJetty.start();
    waitForState("Replica not added", collectionName, activeReplicaCount(0, 2, 0));
    waitForNumDocsInAllActiveReplicas(2);
  }
 

static JettySolrRunner createAndStartJetty(SolrInstance instance) throws Exception {
  FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));
  Properties nodeProperties = new Properties();
  nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
  JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").setPort(0).build();
  JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, jettyConfig);
  jetty.start();
  return jetty;
}
 

/**
 * Starts jetty if its not already running
 */
protected static void start(JettySolrRunner jsr) throws Exception
{
    if (!jsr.isRunning())
    {
        jsr.start();
    }
}
 

public void testKillPullReplica() throws Exception {
    CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 1)
      .setMaxShardsPerNode(100)
      .process(cluster.getSolrClient());
//    cluster.getSolrClient().getZkStateReader().registerCore(collectionName); //TODO: Is this needed?
    waitForState("Expected collection to be created with 1 shard and 2 replicas", collectionName, clusterShape(1, 2));
    DocCollection docCollection = assertNumberOfReplicas(1, 0, 1, false, true);
    assertEquals(1, docCollection.getSlices().size());

    waitForNumDocsInAllActiveReplicas(0);
    cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
    cluster.getSolrClient().commit(collectionName);
    waitForNumDocsInAllActiveReplicas(1);

    JettySolrRunner pullReplicaJetty = cluster.getReplicaJetty(docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0));
    pullReplicaJetty.stop();
    waitForState("Replica not removed", collectionName, activeReplicaCount(1, 0, 0));
    // Also wait for the replica to be placed in state="down"
    waitForState("Didn't update state", collectionName, clusterStateReflectsActiveAndDownReplicas());

    cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "bar"));
    cluster.getSolrClient().commit(collectionName);
    waitForNumDocsInAllActiveReplicas(2);

    pullReplicaJetty.start();
    waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 1));
    waitForNumDocsInAllActiveReplicas(2);
  }
 

@Override
protected void createServers(int numShards) throws Exception {

  System.setProperty("configSetBaseDir", TEST_HOME());

  File controlHome = testDir.toPath().resolve("control").toFile();

  seedSolrHome(controlHome);
  writeCoreProperties(controlHome.toPath().resolve("cores").resolve(DEFAULT_TEST_CORENAME), DEFAULT_TEST_CORENAME);
  controlJetty = createJetty(controlHome, testDir + "/control/data", null, getSolrConfigFile(), getSchemaFile());
  controlJetty.start();
  controlClient = createNewSolrClient(controlJetty.getLocalPort());

  shardsArr = new String[numShards];
  StringBuilder sb = new StringBuilder();
  for (int i = 0; i < numShards; i++) {
    if (sb.length() > 0) sb.append(',');
    String shardname = "shard" + i;
    Path shardHome = testDir.toPath().resolve(shardname);
    seedSolrHome(shardHome.toFile());
    Path coresPath = shardHome.resolve("cores");
    writeCoreProperties(coresPath.resolve(DEFAULT_TEST_CORENAME), DEFAULT_TEST_CORENAME);
    JettySolrRunner j = createJetty(shardHome.toFile(),
        testDir + "/shard" + i + "/data", null, getSolrConfigFile(),
        getSchemaFile());
    j.start();
    jettys.add(j);
    clients.add(createNewSolrClient(j.getLocalPort()));
    String shardStr = buildUrl(j.getLocalPort());
    shardsArr[i] = shardStr;
    sb.append(shardStr);
  }

  shards = sb.toString();
}
 

@BeforeClass
public static void beforeSolrJettyTestBase() throws Exception {
	final File origSolrHome = new File(SolrJettyTestBase.class.getResource("/solr").toURI());
	final File tempSolrHome = tempSolrFolder.getRoot();
	final File tempSolrData = tempSolrFolder.newFolder("data");

	FileUtils.copyDirectory(origSolrHome, tempSolrHome);

	final JettyConfig jettyConfig = JettyConfig.builder()
		.setContext("/solr")
		.setPort(8888)
		.stopAtShutdown(true)
		.build();

	final Properties nodeProperties = new Properties();

	nodeProperties.setProperty("solr.data.dir", tempSolrData.getCanonicalPath());
	nodeProperties.setProperty("coreRootDirectory", tempSolrHome.toString());
	nodeProperties.setProperty("configSetBaseDir", tempSolrHome.toString());

	System.setProperty("jetty.testMode", "true");

	jetty = new JettySolrRunner(tempSolrHome.toString(), nodeProperties, jettyConfig);
	jetty.start();

	client = createNewSolrClient();
}
 

public void testBasicLeaderElection() throws Exception {
  createAndWaitForCollection(1,0,2,0);
  CloudSolrClient cloudClient = cluster.getSolrClient();
  new UpdateRequest()
      .deleteByQuery("*:*")
      .commit(cluster.getSolrClient(), collectionName);
  new UpdateRequest()
      .add(sdoc("id", "1"))
      .add(sdoc("id", "2"))
      .process(cloudClient, collectionName);
  JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
  oldLeaderJetty.stop();
  waitForState("Replica not removed", collectionName, activeReplicaCount(0, 1, 0));

  // Even after the replica is gone, a leader may not be elected yet. Wait for it. 
  waitForLeaderChange(oldLeaderJetty, "shard1");
  
  new UpdateRequest()   
      .add(sdoc("id", "3"))
      .add(sdoc("id", "4"))
      .process(cloudClient, collectionName);
  oldLeaderJetty.start();
  waitForState("Replica not added", collectionName, activeReplicaCount(0, 2, 0));
  checkRTG(1,4, cluster.getJettySolrRunners());
  new UpdateRequest()
      .commit(cloudClient, collectionName);
  waitForNumDocsInAllActiveReplicas(4, 0);
}
 

@Override
public void tearDown() throws Exception {
  for (JettySolrRunner jetty:cluster.getJettySolrRunners()) {
    if (!jetty.isRunning()) {
      log.warn("Jetty {} not running, probably some bad test. Starting it", jetty.getLocalPort());
      jetty.start();
    }
  }
  if (cluster.getSolrClient().getZkStateReader().getClusterState().getCollectionOrNull(collectionName) != null) {
    log.info("tearDown deleting collection");
    CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
    waitForDeletion(collectionName);
  }
  super.tearDown();
}
 

@Override
protected void createServers(int numShards) throws Exception {
  // give everyone there own solrhome
  File controlHome = new File(new File(getSolrHome()).getParentFile(), "control" + homeCount.incrementAndGet());
  FileUtils.copyDirectory(new File(getSolrHome()), controlHome);
  setupJettySolrHome(controlHome);

  controlJetty = createJetty(controlHome, null);      // let the shardId default to shard1
  controlJetty.start();
  controlClient = createNewSolrClient(controlJetty.getLocalPort());

  assertTrue(CollectionAdminRequest
      .createCollection("control_collection", 1, 1)
      .setCreateNodeSet(controlJetty.getNodeName())
      .process(controlClient).isSuccess());

  ZkStateReader zkStateReader = jettys.get(0).getCoreContainer().getZkController()
      .getZkStateReader();

  waitForRecoveriesToFinish("control_collection", zkStateReader, false, true, 15);

  StringBuilder sb = new StringBuilder();
  for (int i = 1; i <= numShards; i++) {
    if (sb.length() > 0) sb.append(',');
    // give everyone there own solrhome
    File jettyHome = new File(new File(getSolrHome()).getParentFile(), "jetty" + homeCount.incrementAndGet());
    setupJettySolrHome(jettyHome);
    JettySolrRunner j = createJetty(jettyHome, null, "shard" + (i + 2));
    j.start();
    jettys.add(j);
    clients.add(createNewSolrClient(j.getLocalPort()));
    sb.append(buildUrl(j.getLocalPort()));
  }

  shards = sb.toString();

}
 

private void doReplaceLeader(boolean removeReplica) throws Exception {
  DocCollection docCollection = createAndWaitForCollection(1, 0, 2, 0);

  // Add a document and commit
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
  cluster.getSolrClient().commit(collectionName);
  Slice s = docCollection.getSlices().iterator().next();
  try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
    assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
  }

  waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);

  // Delete leader replica from shard1
  JettySolrRunner leaderJetty = null;
  if (removeReplica) {
    CollectionAdminRequest.deleteReplica(
        collectionName,
        "shard1",
        s.getLeader().getName())
    .process(cluster.getSolrClient());
  } else {
    leaderJetty = cluster.getReplicaJetty(s.getLeader());
    leaderJetty.stop();
    waitForState("Leader replica not removed", collectionName, clusterShape(1, 1));
    // Wait for cluster state to be updated
    waitForState("Replica state not updated in cluster state",
        collectionName, clusterStateReflectsActiveAndDownReplicas());
  }
  docCollection = assertNumberOfReplicas(0, 1, 0, true, true);

  // Wait until a new leader is elected
  waitForLeaderChange(leaderJetty, "shard1");
  
  // There is a new leader, I should be able to add and commit
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"));
  cluster.getSolrClient().commit(collectionName);

  // Queries should still work
  waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
  // Start back the node
  if (removeReplica) {
    addReplicaWithRetries();
    
  } else {
    leaderJetty.start();
  }
  waitForState("Expected collection to be 1x2", collectionName, clusterShape(1, 2));
  // added replica should replicate from the leader
  waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
}
 

@BeforeClass
public static void beforeTest() throws Exception {
  File homeDir = createTempDir().toFile();

  File collDir = new File(homeDir, "collection1");
  File dataDir = new File(collDir, "data");
  File confDir = new File(collDir, "conf");

  homeDir.mkdirs();
  collDir.mkdirs();
  dataDir.mkdirs();
  confDir.mkdirs();

  FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(homeDir, "solr.xml"));
  String src_dir = TEST_HOME() + "/collection1/conf";
  FileUtils.copyFile(new File(src_dir, "schema-tiny.xml"), 
                     new File(confDir, "schema.xml"));
  FileUtils.copyFile(new File(src_dir, "solrconfig-solcoreproperties.xml"), 
                     new File(confDir, "solrconfig.xml"));
  FileUtils.copyFile(new File(src_dir, "solrconfig.snippet.randomindexconfig.xml"), 
                     new File(confDir, "solrconfig.snippet.randomindexconfig.xml"));

  Properties p = new Properties();
  p.setProperty("foo.foo1", "f1");
  p.setProperty("foo.foo2", "f2");
  Writer fos = new OutputStreamWriter(new FileOutputStream(new File(confDir, "solrcore.properties")), StandardCharsets.UTF_8);
  p.store(fos, null);
  IOUtils.close(fos);

  Files.createFile(collDir.toPath().resolve("core.properties"));


  Properties nodeProperties = new Properties();
  // this sets the property for jetty starting SolrDispatchFilter
  if (System.getProperty("solr.data.dir") == null && System.getProperty("solr.hdfs.home") == null) {
    nodeProperties.setProperty("solr.data.dir", createTempDir().toFile().getCanonicalPath());
  }
  jetty = new JettySolrRunner(homeDir.getAbsolutePath(), nodeProperties, buildJettyConfig("/solr"));

  jetty.start();
  port = jetty.getLocalPort();

  //createJetty(homeDir.getAbsolutePath(), null, null);
}
 

private void testDoRecoveryOnRestart() throws Exception {
  String testCollectionName = "collDoRecoveryOnRestart";
  try {
    // Inject pausing in recovery op, hence the replica won't be able to finish recovery

    TestInjection.prepRecoveryOpPauseForever = "true:100";
    
    createCollection(testCollectionName, "conf1", 1, 2, 1);
    cloudClient.setDefaultCollection(testCollectionName);

    sendDoc(1, 2);

    JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(getShardLeader(testCollectionName, "shard1", 1000)));
    List<Replica> notLeaders =
        ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, maxWaitSecsToSeeAllActive);
    assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 1);

    SocketProxy proxy0 = getProxyForReplica(notLeaders.get(0));
    SocketProxy leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));

    proxy0.close();
    leaderProxy.close();

    // indexing during a partition
    int achievedRf = sendDoc(2, 1, leaderJetty);
    assertEquals("Unexpected achieved replication factor", 1, achievedRf);
    try (ZkShardTerms zkShardTerms = new ZkShardTerms(testCollectionName, "shard1", cloudClient.getZkStateReader().getZkClient())) {
      assertFalse(zkShardTerms.canBecomeLeader(notLeaders.get(0).getName()));
    }
    waitForState(testCollectionName, notLeaders.get(0).getName(), DOWN, 10000);

    // heal partition
    proxy0.reopen();
    leaderProxy.reopen();

    waitForState(testCollectionName, notLeaders.get(0).getName(), RECOVERING, 10000);

    System.clearProperty("solrcloud.skip.autorecovery");
    JettySolrRunner notLeaderJetty = getJettyOnPort(getReplicaPort(notLeaders.get(0)));
    String notLeaderNodeName = notLeaderJetty.getNodeName();
    notLeaderJetty.stop();
    
    cloudClient.getZkStateReader().waitForLiveNodes(15, TimeUnit.SECONDS, SolrCloudTestCase.missingLiveNode(notLeaderNodeName));

    notLeaderJetty.start();
    ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, 130);
    assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 2);
  } finally {
    TestInjection.prepRecoveryOpPauseForever = null;
    TestInjection.notifyPauseForeverDone();
  }

  // try to clean up
  attemptCollectionDelete(cloudClient, testCollectionName);
}
 

/**
 * Test that basic autoAddReplicaLogic kicks in when a node is lost 
 */
@Test
public void testSimple() throws Exception {
  final String COLLECTION = "test_simple";
  final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
  final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
  final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);
  if (log.isInfoEnabled()) {
    log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
        jetty1.getNodeName(), jetty1.getLocalPort(),
        jetty2.getNodeName(), jetty2.getLocalPort());
  }
           
  CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
    .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
    .setAutoAddReplicas(true)
    .setMaxShardsPerNode(2)
    .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(COLLECTION, 2, 4);
  
  // start the tests
  JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
  String lostNodeName = lostJetty.getNodeName();
  List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);
  if (log.isInfoEnabled()) {
    log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.stop();
  
  cluster.waitForJettyToStop(lostJetty);
  waitForNodeLeave(lostNodeName);
  
  waitForState(COLLECTION + "=(2,4) w/o down replicas",
               COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
               
  checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);

  if (log.isInfoEnabled()) {
    log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.start();
  
  waitForNodeLive(lostJetty);
  
  assertTrue("Timeout waiting for all live and active",
             ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));

}
 

@Test
public void testRepFactor1LeaderStartup() throws Exception {

  CloudSolrClient solrClient = cluster.getSolrClient();

  String collectionName = "testRepFactor1LeaderStartup";
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
      .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName());
  create.process(solrClient);

 cluster.waitForActiveCollection(collectionName, 1, 1);

  solrClient.setDefaultCollection(collectionName);

  String addListenerCommand = "{" +
      "'add-listener' : {'name':'newSearcherListener','event':'newSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "'add-listener' : {'name':'firstSearcherListener','event':'firstSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "}";

  ConfigRequest request = new ConfigRequest(addListenerCommand);
  solrClient.request(request);

  solrClient.add(new SolrInputDocument("id", "1"));
  solrClient.commit();

  AtomicInteger expectedDocs = new AtomicInteger(1);
  AtomicReference<String> failingCoreNodeName = new AtomicReference<>();
  CollectionStateWatcher stateWatcher = createActiveReplicaSearcherWatcher(expectedDocs, failingCoreNodeName);

  JettySolrRunner runner = cluster.getJettySolrRunner(0);
  runner.stop();
  
  cluster.waitForJettyToStop(runner);
  // check waitForState only after we are sure the node has shutdown and have forced an update to liveNodes
  // ie: workaround SOLR-13490
  cluster.getSolrClient().getZkStateReader().updateLiveNodes();
  waitForState("jetty count:" + cluster.getJettySolrRunners().size(), collectionName, clusterShape(1, 0));
  
  // restart
  sleepTime.set(1000);
  runner.start();
  cluster.waitForAllNodes(30);
  cluster.getSolrClient().getZkStateReader().registerCollectionStateWatcher(collectionName, stateWatcher);
  cluster.waitForActiveCollection(collectionName, 1, 1);
  assertNull("No replica should have been active without registering a searcher, found: " + failingCoreNodeName.get(), failingCoreNodeName.get());
  cluster.getSolrClient().getZkStateReader().removeCollectionStateWatcher(collectionName, stateWatcher);
}
 

/**
 * Add a previously stopped node back to the cluster
 * @param jetty a {@link JettySolrRunner} previously returned by {@link #stopJettySolrRunner(int)}
 * @return the started node
 * @throws Exception on error
 */
public JettySolrRunner startJettySolrRunner(JettySolrRunner jetty) throws Exception {
  jetty.start(false);
  if (!jettys.contains(jetty)) jettys.add(jetty);
  return jetty;
}