Java Code Examples for org.apache.solr.common.cloud.DocCollection#getSlices()

/**
 * Send request to all replicas of a collection
 * @return List of replicas which is not live for receiving the request
 */
List<Replica> collectionCmd(ZkNodeProps message, ModifiableSolrParams params,
                   NamedList<Object> results, Replica.State stateMatcher, String asyncId, Set<String> okayExceptions) {
  log.info("Executing Collection Cmd={}, asyncId={}", params, asyncId);
  String collectionName = message.getStr(NAME);
  @SuppressWarnings("deprecation")
  ShardHandler shardHandler = shardHandlerFactory.getShardHandler(overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient());

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection coll = clusterState.getCollection(collectionName);
  List<Replica> notLivesReplicas = new ArrayList<>();
  final ShardRequestTracker shardRequestTracker = new ShardRequestTracker(asyncId);
  for (Slice slice : coll.getSlices()) {
    notLivesReplicas.addAll(shardRequestTracker.sliceCmd(clusterState, params, stateMatcher, slice, shardHandler));
  }

  shardRequestTracker.processResponses(results, shardHandler, false, null, okayExceptions);
  return notLivesReplicas;
}
 

public static Map<String, String> mapReplicasToReplicaType(DocCollection collection) {
  Map<String, String> replicaTypeMap = new HashMap<>();
  for (Slice slice : collection.getSlices()) {
    for (Replica replica : slice.getReplicas()) {
      String coreUrl = replica.getCoreUrl();
      // It seems replica reports its core URL with a trailing slash while shard
      // info returned from the query doesn't. Oh well. We will include both, just in case
      replicaTypeMap.put(coreUrl, replica.getType().toString());
      if (coreUrl.endsWith("/")) {
        replicaTypeMap.put(coreUrl.substring(0, coreUrl.length() - 1), replica.getType().toString());
      }else {
        replicaTypeMap.put(coreUrl + "/", replica.getType().toString());
      }
    }
  }
  return replicaTypeMap;
}
 

private void verifyPropCorrectlyDistributed(String prop) throws KeeperException, InterruptedException {

    TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);

    String propLC = prop.toLowerCase(Locale.ROOT);
    DocCollection docCollection = null;
    while (timeout.hasTimedOut() == false) {
      forceUpdateCollectionStatus();
      docCollection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);
      int maxPropCount = Integer.MAX_VALUE;
      int minPropCount = Integer.MIN_VALUE;
      for (Slice slice : docCollection.getSlices()) {
        int repCount = 0;
        for (Replica rep : slice.getReplicas()) {
          if (rep.getBool("property." + propLC, false)) {
            repCount++;
          }
        }
        maxPropCount = Math.max(maxPropCount, repCount);
        minPropCount = Math.min(minPropCount, repCount);
      }
      if (Math.abs(maxPropCount - minPropCount) < 2) return;
    }
    log.error("Property {} is not distributed evenly. {}", prop, docCollection);
    fail("Property is not distributed evenly " + prop);
  }
 

private void checkLeaderStatus() throws InterruptedException, KeeperException {
  for (int idx = 0; pendingOps.size() > 0 && idx < 600; ++idx) {
    ClusterState clusterState = coreContainer.getZkController().getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();
    DocCollection dc = clusterState.getCollection(collectionName);
    for (Slice slice : dc.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        if (replica.isActive(liveNodes) && replica.getBool(SliceMutator.PREFERRED_LEADER_PROP, false)) {
          if (replica.getBool(LEADER_PROP, false)) {
            if (pendingOps.containsKey(slice.getName())) {
              // Record for return that the leader changed successfully
              pendingOps.remove(slice.getName());
              addToSuccesses(slice, replica);
              break;
            }
          }
        }
      }
    }
    TimeUnit.MILLISECONDS.sleep(100);
    coreContainer.getZkController().getZkStateReader().forciblyRefreshAllClusterStateSlow();
  }
  addAnyFailures();
}
 

/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
protected List<SolrCmdDistributor.Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
  Collection<Slice> allSlices = coll.getSlices();
  List<SolrCmdDistributor.Node> nodes = null;
  for (Slice aslice : allSlices) {
    final Slice.State state = aslice.getState();
    if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY)  {
      DocRouter.Range myRange = coll.getSlice(shardId).getRange();
      if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
      if (isSubset &&
          (docId == null // in case of deletes
              || coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll))) {
        Replica sliceLeader = aslice.getLeader();
        // slice leader can be null because node/shard is created zk before leader election
        if (sliceLeader != null && clusterState.liveNodesContain(sliceLeader.getNodeName()))  {
          if (nodes == null) nodes = new ArrayList<>();
          ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(sliceLeader);
          nodes.add(new SolrCmdDistributor.StdNode(nodeProps, coll.getName(), aslice.getName()));
        }
      }
    }
  }
  return nodes;
}
 

private void checkAllReplicasActive() throws KeeperException, InterruptedException {
  TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
  while (timeout.hasTimedOut() == false) {
    forceUpdateCollectionStatus();
    DocCollection docCollection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);
    Set<String> liveNodes = cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes();
    boolean allActive = true;
    for (Slice slice : docCollection.getSlices()) {
      for (Replica rep : slice.getReplicas()) {
        if (rep.isActive(liveNodes) == false) {
          allActive = false;
        }
      }
    }
    if (allActive) {
      return;
    }
    TimeUnit.MILLISECONDS.sleep(100);
  }
  fail("timed out waiting for all replicas to become active");
}
 

long getNumDocs(CloudSolrClient client) throws Exception {
  String collectionName = client.getDefaultCollection();
  DocCollection collection = client.getZkStateReader().getClusterState().getCollection(collectionName);
  Collection<Slice> slices = collection.getSlices();

  long totCount = 0;
  for (Slice slice : slices) {
    if (!slice.getState().equals(Slice.State.ACTIVE)) continue;
    long lastReplicaCount = -1;
    for (Replica replica : slice.getReplicas()) {
      SolrClient replicaClient = getHttpSolrClient(replica.getBaseUrl() + "/" + replica.getCoreName());
      long numFound = 0;
      try {
        numFound = replicaClient.query(params("q", "*:*", "distrib", "false")).getResults().getNumFound();
        log.info("Replica count={} for {}", numFound, replica);
      } finally {
        replicaClient.close();
      }
      if (lastReplicaCount >= 0) {
        assertEquals("Replica doc count for " + replica, lastReplicaCount, numFound);
      }
      lastReplicaCount = numFound;
    }
    totCount += lastReplicaCount;
  }


  long cloudClientDocs = client.query(new SolrQuery("*:*")).getResults().getNumFound();
  assertEquals("Sum of shard count should equal distrib query doc count", totCount, cloudClientDocs);
  return totCount;
}
 

/**
 * @param shardVsReplicaCount shard names vs no:of replicas required for each of those shards
 * @param snitches            snitches details
 * @param shardVsNodes        The current state of the system. can be an empty map if no nodes
 *                            are created in this collection till now
 */
@SuppressWarnings({"unchecked"})
public ReplicaAssigner(List<Rule> rules,
                       Map<String, Integer> shardVsReplicaCount,
                       @SuppressWarnings({"rawtypes"})List snitches,
                       Map<String, Map<String, Integer>> shardVsNodes,
                       List<String> participatingLiveNodes,
                       SolrCloudManager cloudManager, ClusterState clusterState) {
  this.rules = rules;
  for (Rule rule : rules) tagNames.add(rule.tag.name);
  this.shardVsReplicaCount = shardVsReplicaCount;
  this.participatingLiveNodes = new ArrayList<>(participatingLiveNodes);
  this.nodeVsTags = getTagsForNodes(cloudManager, snitches);
  this.shardVsNodes = getDeepCopy(shardVsNodes, 2);

  if (clusterState != null) {
    Map<String, DocCollection> collections = clusterState.getCollectionsMap();
    for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
      DocCollection coll = entry.getValue();
      for (Slice slice : coll.getSlices()) {
        for (Replica replica : slice.getReplicas()) {
          AtomicInteger count = nodeVsCores.get(replica.getNodeName());
          if (count == null) nodeVsCores.put(replica.getNodeName(), count = new AtomicInteger());
          count.incrementAndGet();
        }
      }
    }
  }
}
 

public static CreateReplica assignReplicaDetails(SolrCloudManager cloudManager, ClusterState clusterState,
                                               ZkNodeProps message, ReplicaPosition replicaPosition) {
  boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);

  String collection = message.getStr(COLLECTION_PROP);
  String node = replicaPosition.node;
  String shard = message.getStr(SHARD_ID_PROP);
  String coreName = message.getStr(CoreAdminParams.NAME);
  String coreNodeName = message.getStr(CoreAdminParams.CORE_NODE_NAME);
  Replica.Type replicaType = replicaPosition.type;

  if (StringUtils.isBlank(coreName)) {
    coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
  }

  log.info("Node Identified {} for creating new replica of shard {} for collection {}", node, shard, collection);
  if (!clusterState.liveNodesContain(node)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
  }
  DocCollection coll = clusterState.getCollection(collection);
  if (coreName == null) {
    coreName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), coll, shard, replicaType);
  } else if (!skipCreateReplicaInClusterState) {
    //Validate that the core name is unique in that collection
    for (Slice slice : coll.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        String replicaCoreName = replica.getStr(CORE_NAME_PROP);
        if (coreName.equals(replicaCoreName)) {
          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" +
              " for this collection");
        }
      }
    }
  }
  log.info("Returning CreateReplica command.");
  return new CreateReplica(collection, shard, node, replicaType, coreName, coreNodeName);
}
 

@Override
public List<ReplicaPosition> assign(SolrCloudManager solrCloudManager, AssignRequest assignRequest) throws Assign.AssignmentException, IOException, InterruptedException {
  if (assignRequest.numTlogReplicas + assignRequest.numPullReplicas != 0) {
    throw new Assign.AssignmentException(
        Replica.Type.TLOG + " or " + Replica.Type.PULL + " replica types not supported with placement rules or cluster policies");
  }

  Map<String, Integer> shardVsReplicaCount = new HashMap<>();
  for (String shard : assignRequest.shardNames) shardVsReplicaCount.put(shard, assignRequest.numNrtReplicas);

  Map<String, Map<String, Integer>> shardVsNodes = new LinkedHashMap<>();
  DocCollection docCollection = solrCloudManager.getClusterStateProvider().getClusterState().getCollectionOrNull(assignRequest.collectionName);
  if (docCollection != null) {
    for (Slice slice : docCollection.getSlices()) {
      LinkedHashMap<String, Integer> n = new LinkedHashMap<>();
      shardVsNodes.put(slice.getName(), n);
      for (Replica replica : slice.getReplicas()) {
        Integer count = n.get(replica.getNodeName());
        if (count == null) count = 0;
        n.put(replica.getNodeName(), ++count);
      }
    }
  }

  List<String> nodesList = assignRequest.nodes == null ? new ArrayList<>(clusterState.getLiveNodes()) : assignRequest.nodes;

  ReplicaAssigner replicaAssigner = new ReplicaAssigner(rules,
      shardVsReplicaCount,
      snitches,
      shardVsNodes,
      nodesList,
      solrCloudManager, clusterState);

  Map<ReplicaPosition, String> nodeMappings = replicaAssigner.getNodeMappings();
  return nodeMappings.entrySet().stream()
      .map(e -> new ReplicaPosition(e.getKey().shard, e.getKey().index, e.getKey().type, e.getValue()))
      .collect(Collectors.toList());
}
 

static HashMap<String, ReplicaCount> getNodeNameVsShardCount(String collectionName,
                                                                     ClusterState clusterState, List<String> createNodeList) {
  HashMap<String, ReplicaCount> nodeNameVsShardCount = new HashMap<>();
  List<String> liveNodes = createNodeList == null || createNodeList.isEmpty() ?
      new ArrayList<>(clusterState.getLiveNodes()) :
      checkLiveNodes(createNodeList, clusterState);

  for (String s : liveNodes) {
    nodeNameVsShardCount.put(s, new ReplicaCount(s));
  }

  // if we were given a list, just use that, don't worry about counts
  if (createNodeList != null) { // Overrides petty considerations about maxShardsPerNode
    return nodeNameVsShardCount;
  }

  // if we get here we were not given a createNodeList, build a map with real counts.
  DocCollection coll = clusterState.getCollection(collectionName);
  int maxShardsPerNode = coll.getMaxShardsPerNode() == -1 ? Integer.MAX_VALUE : coll.getMaxShardsPerNode();
  Map<String, DocCollection> collections = clusterState.getCollectionsMap();
  for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
    DocCollection c = entry.getValue();
    //identify suitable nodes  by checking the no:of cores in each of them
    for (Slice slice : c.getSlices()) {
      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        ReplicaCount count = nodeNameVsShardCount.get(replica.getNodeName());
        if (count != null) {
          count.totalNodes++; // Used to "weigh" whether this node should be used later.
          if (entry.getKey().equals(collectionName)) {
            count.thisCollectionNodes++;
            if (count.thisCollectionNodes >= maxShardsPerNode) nodeNameVsShardCount.remove(replica.getNodeName());
          }
        }
      }
    }
  }

  return nodeNameVsShardCount;
}
 

protected static boolean assertShardInSync(String collection, String shard, CloudSolrClient client) throws IOException, SolrServerException {
  TimeOut waitTimeOut = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  DocCollection docCollection = client.getZkStateReader().getClusterState().getCollection(collection);
  Slice correctSlice = null;
  for (Slice slice : docCollection.getSlices()) {
    if (shard.equals(slice.getName())) {
      correctSlice = slice;
      break;
    }
  }
  assertNotNull(correctSlice);

  long leaderDocCount;
  try (HttpSolrClient leaderClient = new HttpSolrClient.Builder(correctSlice.getLeader().getCoreUrl()).withHttpClient(client.getHttpClient()).build()) {
    leaderDocCount = leaderClient.query(new SolrQuery("*:*").setParam("distrib", "false")).getResults().getNumFound();
  }

  while (!waitTimeOut.hasTimedOut()) {
    int replicasInSync = 0;
    for (Replica replica : correctSlice.getReplicas()) {
      try (HttpSolrClient leaderClient = new HttpSolrClient.Builder(replica.getCoreUrl()).withHttpClient(client.getHttpClient()).build()) {
        long replicaDocCount = leaderClient.query(new SolrQuery("*:*").setParam("distrib", "false")).getResults().getNumFound();
        if (replicaDocCount == leaderDocCount) replicasInSync++;
      }
    }
    if (replicasInSync == correctSlice.getReplicas().size()) {
      return true;
    }
  }
  return false;
}
 

public static String getRandomServer(CloudSolrClient cloudClient, String collName) {
  DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection(collName);
  List<String> urls = new ArrayList<>();
  for (Slice slice : coll.getSlices()) {
    for (Replica replica : slice.getReplicas())
      urls.add(""+replica.get(ZkStateReader.BASE_URL_PROP) + "/"+replica.get(ZkStateReader.CORE_NAME_PROP));
  }
  return urls.get(random().nextInt(urls.size()));
}
 

private void checkConfReload(SolrZkClient client, String resPath, String name, String uri) throws Exception {
  Stat stat =  new Stat();
  byte[] data = null;
  try {
    data = client.getData(resPath, null, stat, true);
  } catch (KeeperException.NoNodeException e) {
    data = "{}".getBytes(StandardCharsets.UTF_8);
    log.info("creating_node {}",resPath);
    client.create(resPath,data, CreateMode.PERSISTENT,true);
  }
  long startTime = System.nanoTime();
  Stat newStat = client.setData(resPath, data, true);
  client.setData("/configs/conf1", new byte[]{1}, true);
  assertTrue(newStat.getVersion() > stat.getVersion());
  if (log.isInfoEnabled()) {
    log.info("new_version {}", newStat.getVersion());
  }
  Integer newVersion = newStat.getVersion();
  long maxTimeoutSeconds = 60;
  DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection("collection1");
  List<String> urls = new ArrayList<>();
  for (Slice slice : coll.getSlices()) {
    for (Replica replica : slice.getReplicas())
      urls.add(""+replica.get(ZkStateReader.BASE_URL_PROP) + "/"+replica.get(ZkStateReader.CORE_NAME_PROP));
  }
  HashSet<String> succeeded = new HashSet<>();

  while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds){
    Thread.sleep(50);
    for (String url : urls) {
      MapWriter respMap = getAsMap(url + uri);
      if (String.valueOf(newVersion).equals(respMap._getStr(asList(name, "znodeVersion"), null))) {
        succeeded.add(url);
      }
    }
    if(succeeded.size() == urls.size()) break;
    succeeded.clear();
  }
  assertEquals(StrUtils.formatString("tried these servers {0} succeeded only in {1} ", urls, succeeded) , urls.size(), succeeded.size());
}
 

/** 
 * Total number of replicas for all shards as indicated by the cluster state, regardless of status.
 *
 * @deprecated This method is virtually useless as it does not consider the status of either
 *             the shard or replica, nor wether the node hosting each replica is alive.
 */
@Deprecated
protected int getTotalReplicas(DocCollection c, String collection) {
  if (c == null) return 0;  // support for when collection hasn't been created yet
  int cnt = 0;
  for (Slice slices : c.getSlices()) {
    cnt += slices.getReplicas().size();
  }
  return cnt;
}
 

private void checkElectionQueues() throws KeeperException, InterruptedException {

    DocCollection docCollection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);
    Set<String> liveNodes = cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes();

    for (Slice slice : docCollection.getSlices()) {
      Set<Replica> liveReplicas = new HashSet<>();
      slice.getReplicas().forEach(replica -> {
        if (replica.isActive(liveNodes)) {
          liveReplicas.add(replica);
        }
      });
      checkOneQueue(docCollection, slice, liveReplicas);
    }
  }
 

@Repeat(iterations=2) // 2 times to make sure cleanup is complete and we can create the same collection
public void testCreateDelete() throws Exception {
  switch (random().nextInt(3)) {
    case 0:
      CollectionAdminRequest.createCollection(collectionName, "conf", 2, 0, 4, 0)
      .setMaxShardsPerNode(100)
      .process(cluster.getSolrClient());
      cluster.waitForActiveCollection(collectionName, 2, 8);
      break;
    case 1:
      // Sometimes don't use SolrJ
      String url = String.format(Locale.ROOT, "%s/admin/collections?action=CREATE&name=%s&collection.configName=%s&numShards=%s&tlogReplicas=%s&maxShardsPerNode=%s",
          cluster.getRandomJetty(random()).getBaseUrl(),
          collectionName, "conf",
          2,    // numShards
          4,    // tlogReplicas
          100); // maxShardsPerNode
      HttpGet createCollectionGet = new HttpGet(url);
      HttpResponse httpResponse = cluster.getSolrClient().getHttpClient().execute(createCollectionGet);
      assertEquals(200, httpResponse.getStatusLine().getStatusCode());
      cluster.waitForActiveCollection(collectionName, 2, 8);
      break;
    case 2:
      // Sometimes use V2 API
      url = cluster.getRandomJetty(random()).getBaseUrl().toString() + "/____v2/c";
      String requestBody = String.format(Locale.ROOT, "{create:{name:%s, config:%s, numShards:%s, tlogReplicas:%s, maxShardsPerNode:%s}}",
          collectionName, "conf",
          2,    // numShards
          4,    // tlogReplicas
          100); // maxShardsPerNode
      HttpPost createCollectionPost = new HttpPost(url);
      createCollectionPost.setHeader("Content-type", "application/json");
      createCollectionPost.setEntity(new StringEntity(requestBody));
      httpResponse = cluster.getSolrClient().getHttpClient().execute(createCollectionPost);
      assertEquals(200, httpResponse.getStatusLine().getStatusCode());
      cluster.waitForActiveCollection(collectionName, 2, 8);
      break;
  }

  boolean reloaded = false;
  while (true) {
    DocCollection docCollection = getCollectionState(collectionName);
    assertNotNull(docCollection);
    assertEquals("Expecting 2 shards",
        2, docCollection.getSlices().size());
    assertEquals("Expecting 4 relpicas per shard",
        8, docCollection.getReplicas().size());
    assertEquals("Expecting 8 tlog replicas, 4 per shard",
        8, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).size());
    assertEquals("Expecting no nrt replicas",
        0, docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).size());
    assertEquals("Expecting no pull replicas",
        0, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).size());
    for (Slice s:docCollection.getSlices()) {
      assertTrue(s.getLeader().getType() == Replica.Type.TLOG);
      List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);
      assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),
          4, shardElectionNodes.size());
    }
    assertUlogPresence(docCollection);
    if (reloaded) {
      break;
    } else {
      // reload
      CollectionAdminResponse response = CollectionAdminRequest.reloadCollection(collectionName)
      .process(cluster.getSolrClient());
      assertEquals(0, response.getStatus());
      waitForState("failed waiting for active colletion", collectionName, clusterShape(2, 8));
      reloaded = true;
    }
  }
}
 

/**
 * Updates the mappings between the jetty's instances and the zookeeper cluster state.
 */
protected void updateMappingsFromZk(String collection) throws Exception {
  List<CloudJettyRunner> cloudJettys = new ArrayList<>();
  Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
  Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();

  CloudSolrClient cloudClient = this.createCloudClient(null);
  try {
    cloudClient.connect();
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(collection);

    for (JettySolrRunner jetty : jettys) {
      int port = jetty.getLocalPort();
      if (port == -1) {
        throw new RuntimeException("Cannot find the port for jetty");
      }

      nextJetty:
      for (Slice shard : coll.getSlices()) {
        Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
        for (Map.Entry<String, Replica> entry : entries) {
          Replica replica = entry.getValue();
          if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
            if (!shardToJetty.containsKey(shard.getName())) {
              shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
            }
            boolean isLeader = shard.getLeader() == replica;
            CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
            shardToJetty.get(shard.getName()).add(cjr);
            if (isLeader) {
              shardToLeaderJetty.put(shard.getName(), cjr);
            }
            cloudJettys.add(cjr);
            break nextJetty;
          }
        }
      }
    }

    List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
    if (oldRunners != null)  {
      // must close resources for the old entries
      for (CloudJettyRunner oldRunner : oldRunners) {
        IOUtils.closeQuietly(oldRunner.client);
      }
    }

    this.cloudJettys.put(collection, cloudJettys);
    this.shardToJetty.put(collection, shardToJetty);
    this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
  } finally {
    cloudClient.close();
  }
}
 

private boolean waitingForReplicasNotLive(ZkStateReader zkStateReader, int timeoutInMs, List<JettySolrRunner> jetties) {
  Set<String> nodeNames = jetties.stream()
      .filter(jetty -> jetty.getCoreContainer() != null)
      .map(JettySolrRunner::getNodeName)
      .collect(Collectors.toSet());
  long timeout = System.nanoTime()
      + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
  boolean success = false;
  while (!success && System.nanoTime() < timeout) {
    success = true;
    ClusterState clusterState = zkStateReader.getClusterState();
    if (clusterState != null) {
      Map<String, DocCollection> collections = clusterState.getCollectionsMap();
      for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
        DocCollection docCollection = entry.getValue();
        Collection<Slice> slices = docCollection.getSlices();
        for (Slice slice : slices) {
          // only look at active shards
          if (slice.getState() == Slice.State.ACTIVE) {
            Collection<Replica> replicas = slice.getReplicas();
            for (Replica replica : replicas) {
              if (nodeNames.contains(replica.getNodeName())) {
                boolean live = clusterState.liveNodesContain(replica
                    .getNodeName());
                if (live) {
                  success = false;
                }
              }
            }
          }
        }
      }
      if (!success) {
        try {
          Thread.sleep(500);
        } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted");
        }
      }
    }
  }

  return success;
}
 

private static void waitForRecovery(String collection) throws Exception {
  log.info("Waiting for recovery of {}", collection);
  boolean recovered = false;
  boolean allActive = true;
  boolean hasLeaders = true;
  DocCollection collState = null;
  for (int i = 0; i < 300; i++) {
    ClusterState state = solrClient.getZkStateReader().getClusterState();
    collState = getCollectionState(collection);
    log.debug("###### {}", collState);
    Collection<Replica> replicas = collState.getReplicas();
    allActive = true;
    hasLeaders = true;
    if (replicas != null && !replicas.isEmpty()) {
      for (Replica r : replicas) {
        if (state.getLiveNodes().contains(r.getNodeName())) {
          if (!r.isActive(state.getLiveNodes())) {
            log.info("Not active: {}", r);
            allActive = false;
          }
        } else {
          log.info("Replica no longer on a live node, ignoring: {}", r);
        }
      }
    } else {
      allActive = false;
    }
    for (Slice slice : collState.getSlices()) {
      if (slice.getLeader() == null) {
        hasLeaders = false;
      }
    }
    if (allActive && hasLeaders) {
      recovered = true;
      break;
    } else {
      log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders);
      Thread.sleep(1000);
    }
  }
  assertTrue("replica never fully recovered: allActive=" + allActive + ", hasLeaders=" + hasLeaders + ", collState=" + collState, recovered);

}