Java Code Examples for org.apache.solr.common.cloud.Slice#getLeader()

protected void logReplicaTypesReplicationInfo(String collectionName, ZkStateReader zkStateReader) throws KeeperException, InterruptedException, IOException {
  log.info("## Collecting extra Replica.Type information of the cluster");
  zkStateReader.updateLiveNodes();
  StringBuilder builder = new StringBuilder();
  zkStateReader.forceUpdateCollection(collectionName);
  DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
  for(Slice s:collection.getSlices()) {
    Replica leader = s.getLeader();
    for (Replica r:s.getReplicas()) {
      if (!r.isActive(zkStateReader.getClusterState().getLiveNodes())) {
        builder.append(String.format(Locale.ROOT, "Replica %s not in liveNodes or is not active%s", r.getName(), System.lineSeparator()));
        continue;
      }
      if (r.equals(leader)) {
        builder.append(String.format(Locale.ROOT, "Replica %s is leader%s", r.getName(), System.lineSeparator()));
      }
      logReplicationDetails(r, builder);
    }
  }
  log.info("Summary of the cluster: {}", builder);
}
 

public List<List<String>> extractShardUrls(String zkHost, String collection) {

    DocCollection docCollection = extractDocCollection(zkHost, collection);
    List<Slice> slices = getSortedSlices(docCollection.getSlices());
    List<List<String>> solrUrls = new ArrayList<List<String>>(slices.size());
    for (Slice slice : slices) {
      if (slice.getLeader() == null) {
        throw new IllegalArgumentException("Cannot find SolrCloud slice leader. "
            + "It looks like not all of your shards are registered in ZooKeeper yet");
      }
      Collection<Replica> replicas = slice.getReplicas();
      List<String> urls = new ArrayList<String>(replicas.size());
      for (Replica replica : replicas) {
        ZkCoreNodeProps props = new ZkCoreNodeProps(replica);
        urls.add(props.getCoreUrl());
      }
      solrUrls.add(urls);
    }
    return solrUrls;
  }
 

/**
 * Method gets the SolrClient for the leader replica. This is needed for a workaround for SOLR-8733.
 */
public SolrClient getClientForLeader() throws KeeperException, InterruptedException {
  ZkStateReader zkStateReader = cloudClient.getZkStateReader();
  cloudClient.getZkStateReader().forceUpdateCollection(DEFAULT_COLLECTION);
  ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
  Replica leader = null;
  Slice shard1 = clusterState.getCollection(DEFAULT_COLLECTION).getSlice(SHARD1);
  leader = shard1.getLeader();

  for (int i = 0; i < clients.size(); i++) {
    String leaderBaseUrl = zkStateReader.getBaseUrlForNodeName(leader.getNodeName());
    if (((HttpSolrClient) clients.get(i)).getBaseURL().startsWith(leaderBaseUrl))
      return clients.get(i);
  }

  return null;
}
 

private List<JettySolrRunner> getSolrRunner(boolean isLeader) {
  List<JettySolrRunner> rs = new ArrayList<>();
  CloudSolrClient cloudClient = cluster.getSolrClient();
  DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(collectionName);
  for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
    if (solrRunner.getCoreContainer() == null) continue;
    for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
      CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
      Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
      Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
      if (slice.getLeader() == replica && isLeader) {
        rs.add(solrRunner);
      } else if (slice.getLeader() != replica && !isLeader) {
        rs.add(solrRunner);
      }
    }
  }
  return rs;
}
 

/**
 * Pick replicas to be deleted. Avoid picking the leader.
 */
private Set<String> pickReplicasTobeDeleted(Slice slice, String shard, String collectionName, int count) {
  validateReplicaAvailability(slice, shard, collectionName, count);
  Collection<Replica> allReplicas = slice.getReplicas();
  Set<String> replicasToBeRemoved = new HashSet<String>();
  Replica leader = slice.getLeader();
  for (Replica replica: allReplicas) {
    if (count == 0) {
      break;
    }
    //Try avoiding to pick up the leader to minimize activity on the cluster.
    if (leader.getCoreName().equals(replica.getCoreName())) {
      continue;
    }
    replicasToBeRemoved.add(replica.getName());
    count --;
  }
  return replicasToBeRemoved;
}
 

/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
protected List<SolrCmdDistributor.Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
  Collection<Slice> allSlices = coll.getSlices();
  List<SolrCmdDistributor.Node> nodes = null;
  for (Slice aslice : allSlices) {
    final Slice.State state = aslice.getState();
    if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY)  {
      DocRouter.Range myRange = coll.getSlice(shardId).getRange();
      if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
      if (isSubset &&
          (docId == null // in case of deletes
              || coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll))) {
        Replica sliceLeader = aslice.getLeader();
        // slice leader can be null because node/shard is created zk before leader election
        if (sliceLeader != null && clusterState.liveNodesContain(sliceLeader.getNodeName()))  {
          if (nodes == null) nodes = new ArrayList<>();
          ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(sliceLeader);
          nodes.add(new SolrCmdDistributor.StdNode(nodeProps, coll.getName(), aslice.getName()));
        }
      }
    }
  }
  return nodes;
}
 

public QueryResponse simQuery(QueryRequest req) throws SolrException, InterruptedException, IOException {
  ensureNotClosed();
  String collection = req.getCollection();
  if (collection == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection not set");
  }
  ensureSystemCollection(collection);
  if (!colShardReplicaMap.containsKey(collection)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection does not exist");
  }
  String query = req.getParams().get(CommonParams.Q);
  if (query == null || !query.equals("*:*")) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only '*:*' query is supported");
  }
  ClusterState clusterState = getClusterState();
  DocCollection coll = clusterState.getCollection(collection);
  AtomicLong count = new AtomicLong();
  for (Slice s : coll.getActiveSlicesArr()) {
    Replica r = s.getLeader();
    if (r == null) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, collection + "/" + s.getName() + " has no leader");
    }
    ReplicaInfo ri = getReplicaInfo(r);
    Number numDocs = (Number)ri.getVariable("SEARCHER.searcher.numDocs", 0L);
    count.addAndGet(numDocs.longValue());
    AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES);
    if (bufferedUpdates != null) {
      count.addAndGet(bufferedUpdates.get());
    }
  }
  QueryResponse rsp = new QueryResponse();
  NamedList<Object> values = new NamedList<>();
  values.add("responseHeader", new NamedList<>());
  SolrDocumentList docs = new SolrDocumentList();
  docs.setNumFound(count.get());
  values.add("response", docs);
  rsp.setResponse(values);
  return rsp;
}
 

private void addProducers(Map<String, Slice> m) {
  for (Map.Entry<String, Slice> entry : m.entrySet()) {
    Slice slice = entry.getValue();
    Replica replica = slice.getLeader();
    if (replica == null) replica = slice.getReplicas().iterator().next();// get a random replica
    CoreHandler coreHandler = new CoreHandler(replica);
    corehandlers.put(replica.getCoreName(), coreHandler);
  }
}
 

private void mapReplicasToClients() throws KeeperException, InterruptedException {
  ZkStateReader zkStateReader = cloudClient.getZkStateReader();
  cloudClient.getZkStateReader().forceUpdateCollection(DEFAULT_COLLECTION);
  ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
  Replica leader = null;
  Slice shard1 = clusterState.getCollection(DEFAULT_COLLECTION).getSlice(SHARD1);
  leader = shard1.getLeader();

  String leaderBaseUrl = zkStateReader.getBaseUrlForNodeName(leader.getNodeName());
  for (int i=0; i<clients.size(); i++) {
    if (((HttpSolrClient)clients.get(i)).getBaseURL().startsWith(leaderBaseUrl))
      LEADER = clients.get(i);
  }
  
  NONLEADERS = new ArrayList<>();
  for (Replica rep: shard1.getReplicas()) {
    if (rep.equals(leader)) {
      continue;
    }
    String baseUrl = zkStateReader.getBaseUrlForNodeName(rep.getNodeName());
    for (int i=0; i<clients.size(); i++) {
      if (((HttpSolrClient)clients.get(i)).getBaseURL().startsWith(baseUrl))
        NONLEADERS.add(clients.get(i));
    }
  }
  
  assertNotNull(LEADER);
  assertEquals(2, NONLEADERS.size());
}
 

private SolrCmdDistributor.Node getLeaderNode(String collection, Slice slice) {
  //TODO when should we do StdNode vs RetryNode?
  final Replica leader = slice.getLeader();
  if (leader == null) {
    throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
        "No 'leader' replica available for shard " + slice.getName() + " of collection " + collection);
  }
  return new SolrCmdDistributor.ForwardNode(new ZkCoreNodeProps(leader), zkController.getZkStateReader(),
      collection, slice.getName(), DistributedUpdateProcessor.MAX_RETRIES_ON_FORWARD_DEAULT);
}
 

@SuppressWarnings({"unchecked"})
private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception {
  String backupName = request.getStr(NAME);
  String asyncId = request.getStr(ASYNC);
  String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
  ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(ocmh.overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient());

  String commitName = request.getStr(CoreAdminParams.COMMIT_NAME);
  Optional<CollectionSnapshotMetaData> snapshotMeta = Optional.empty();
  if (commitName != null) {
    SolrZkClient zkClient = ocmh.zkStateReader.getZkClient();
    snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
    if (!snapshotMeta.isPresent()) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
          + " does not exist for collection " + collectionName);
    }
    if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName
          + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
    }
  }

  log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
      backupPath);

  Collection<String> shardsToConsider = Collections.emptySet();
  if (snapshotMeta.isPresent()) {
    shardsToConsider = snapshotMeta.get().getShards();
  }

  final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
  for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
    Replica replica = null;

    if (snapshotMeta.isPresent()) {
      if (!shardsToConsider.contains(slice.getName())) {
        log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.",
            slice.getName(), collectionName, snapshotMeta.get().getName());
        continue;
      }
      replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
    } else {
      // Note - Actually this can return a null value when there is no leader for this shard.
      replica = slice.getLeader();
      if (replica == null) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
      }
    }

    String coreName = replica.getStr(CORE_NAME_PROP);

    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
    params.set(NAME, slice.getName());
    params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
    params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
    params.set(CORE_NAME_PROP, coreName);
    if (snapshotMeta.isPresent()) {
      params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
    }

    shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);
    log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
  }
  log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);

  shardRequestTracker.processResponses(results, shardHandler, true, "Could not backup all shards");
}
 

/**
 * Return a {@link CollectionStatePredicate} that returns true if a collection has the expected
 * number of shards and replicas.
 * <p>Note: for shards marked as inactive the current Solr behavior is that replicas remain active.
 * {@link org.apache.solr.cloud.autoscaling.sim.SimCloudManager} follows this behavior.</p>
 * @param expectedShards expected number of shards
 * @param expectedReplicas expected number of active replicas per shard
 * @param withInactive if true then count also inactive shards
 * @param requireLeaders if true then require that each shard has a leader
 */
public static CollectionStatePredicate clusterShape(int expectedShards, int expectedReplicas, boolean withInactive,
                                                    boolean requireLeaders) {
  return (liveNodes, collectionState) -> {
    if (collectionState == null) {
      log.debug("-- null collection");
      return false;
    }
    Collection<Slice> slices = withInactive ? collectionState.getSlices() : collectionState.getActiveSlices();
    if (slices.size() != expectedShards) {
      if (log.isDebugEnabled()) {
        log.debug("-- wrong number of slices for collection {}, expected={}, found={}: {}", collectionState.getName(), expectedShards, collectionState.getSlices().size(), collectionState.getSlices());
      }
      return false;
    }
    Set<String> leaderless = new HashSet<>();
    for (Slice slice : slices) {
      int activeReplicas = 0;
      if (requireLeaders && slice.getState() != Slice.State.INACTIVE && slice.getLeader() == null) {
        leaderless.add(slice.getName());
        continue;
      }
      // skip other checks, we're going to fail anyway
      if (!leaderless.isEmpty()) {
        continue;
      }
      for (Replica replica : slice) {
        if (replica.isActive(liveNodes))
          activeReplicas++;
      }
      if (activeReplicas != expectedReplicas) {
        if (log.isDebugEnabled()) {
          log.debug("-- wrong number of active replicas for collection {} in slice {}, expected={}, found={}", collectionState.getName(), slice.getName(), expectedReplicas, activeReplicas);
        }
        return false;
      }
    }
    if (leaderless.isEmpty()) {
      return true;
    } else {
      log.info("-- shards without leaders: {}", leaderless);
      return false;
    }
  };
}
 

private static void waitForRecovery(String collection) throws Exception {
  log.info("Waiting for recovery of {}", collection);
  boolean recovered = false;
  boolean allActive = true;
  boolean hasLeaders = true;
  DocCollection collState = null;
  for (int i = 0; i < 300; i++) {
    ClusterState state = solrClient.getZkStateReader().getClusterState();
    collState = getCollectionState(collection);
    log.debug("###### {}", collState);
    Collection<Replica> replicas = collState.getReplicas();
    allActive = true;
    hasLeaders = true;
    if (replicas != null && !replicas.isEmpty()) {
      for (Replica r : replicas) {
        if (state.getLiveNodes().contains(r.getNodeName())) {
          if (!r.isActive(state.getLiveNodes())) {
            log.info("Not active: {}", r);
            allActive = false;
          }
        } else {
          log.info("Replica no longer on a live node, ignoring: {}", r);
        }
      }
    } else {
      allActive = false;
    }
    for (Slice slice : collState.getSlices()) {
      if (slice.getLeader() == null) {
        hasLeaders = false;
      }
    }
    if (allActive && hasLeaders) {
      recovered = true;
      break;
    } else {
      log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders);
      Thread.sleep(1000);
    }
  }
  assertTrue("replica never fully recovered: allActive=" + allActive + ", hasLeaders=" + hasLeaders + ", collState=" + collState, recovered);

}
 

/**
 * Updates the mappings between the jetty's instances and the zookeeper cluster state.
 */
protected void updateMappingsFromZk(String collection) throws Exception {
  List<CloudJettyRunner> cloudJettys = new ArrayList<>();
  Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
  Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();

  CloudSolrClient cloudClient = this.createCloudClient(null);
  try {
    cloudClient.connect();
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(collection);

    for (JettySolrRunner jetty : jettys) {
      int port = jetty.getLocalPort();
      if (port == -1) {
        throw new RuntimeException("Cannot find the port for jetty");
      }

      nextJetty:
      for (Slice shard : coll.getSlices()) {
        Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
        for (Map.Entry<String, Replica> entry : entries) {
          Replica replica = entry.getValue();
          if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
            if (!shardToJetty.containsKey(shard.getName())) {
              shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
            }
            boolean isLeader = shard.getLeader() == replica;
            CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
            shardToJetty.get(shard.getName()).add(cjr);
            if (isLeader) {
              shardToLeaderJetty.put(shard.getName(), cjr);
            }
            cloudJettys.add(cjr);
            break nextJetty;
          }
        }
      }
    }

    List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
    if (oldRunners != null)  {
      // must close resources for the old entries
      for (CloudJettyRunner oldRunner : oldRunners) {
        IOUtils.closeQuietly(oldRunner.client);
      }
    }

    this.cloudJettys.put(collection, cloudJettys);
    this.shardToJetty.put(collection, shardToJetty);
    this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
  } finally {
    cloudClient.close();
  }
}
 

@Nightly
public void testVeryLargeCluster() throws Exception {
  String COLLECTION_NAME = "veryLargeColl";
  configureCluster(4)
      .addConfig("conf", configset("cloud-minimal"))
      .configure();

  try {
    CollectionAdminRequest
        .createCollection(COLLECTION_NAME, "conf", 8, 1)
        .setMaxShardsPerNode(10)
        .process(cluster.getSolrClient());
    cluster.waitForActiveCollection(COLLECTION_NAME, 8, 8);

    String tmpFileLoc = new File(cluster.getBaseDir().toFile().getAbsolutePath() +
        File.separator).getPath();
    String url = cluster.getRandomJetty(random()).getBaseUrl() + "/" + COLLECTION_NAME;


    int docCount = 0;

    for (int j = 0; j < 4; j++) {
      int bsz = 10000;
      UpdateRequest ur = new UpdateRequest();
      ur.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
      for (int i = 0; i < bsz; i++) {
        ur.add("id", String.valueOf((j * bsz) + i), "desc_s", TestUtil.randomSimpleString(random(), 10, 50));
      }
      cluster.getSolrClient().request(ur, COLLECTION_NAME);
      docCount += bsz;
    }

    QueryResponse qr = cluster.getSolrClient().query(COLLECTION_NAME, new SolrQuery("*:*").setRows(0));
    assertEquals(docCount, qr.getResults().getNumFound());

    DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getCollection(COLLECTION_NAME);
    HashMap<String, Long> docCounts = new HashMap<>();
    long totalDocsFromCores = 0;
    for (Slice slice : coll.getSlices()) {
      Replica replica = slice.getLeader();
      try (HttpSolrClient client = new HttpSolrClient.Builder(replica.getBaseUrl()).build()) {
        long count = ExportTool.getDocCount(replica.getCoreName(), client);
        docCounts.put(replica.getCoreName(), count);
        totalDocsFromCores += count;
      }
    }
    assertEquals(docCount, totalDocsFromCores);

    ExportTool.MultiThreadedRunner info = null;
    String absolutePath = null;

    info = new ExportTool.MultiThreadedRunner(url);
    info.output = System.out;
    absolutePath = tmpFileLoc + COLLECTION_NAME + random().nextInt(100000) + ".javabin";
    info.setOutFormat(absolutePath, "javabin");
    info.setLimit("-1");
    info.exportDocs();
    assertJavabinDocsCount(info, docCount);
    for (Map.Entry<String, Long> e : docCounts.entrySet()) {
      assertEquals(e.getValue().longValue(), info.corehandlers.get(e.getKey()).receivedDocs.get());
    }
    info = new ExportTool.MultiThreadedRunner(url);
    info.output = System.out;
    absolutePath = tmpFileLoc + COLLECTION_NAME + random().nextInt(100000) + ".json";
    info.setOutFormat(absolutePath, "jsonl");
    info.fields = "id,desc_s";
    info.setLimit("-1");
    info.exportDocs();
    long actual = ((ExportTool.JsonSink) info.sink).info.docsWritten.get();
    assertTrue("docs written :" + actual + "docs produced : " + info.docsWritten.get(), actual >= docCount);
    assertJsonDocsCount(info, docCount,null);
  } finally {
    cluster.shutdown();

  }
}
 

protected static void ensureAllReplicasAreActive(String testCollectionName, int shards, int rf, int maxWaitSecs)
    throws Exception {
  long startMs = System.currentTimeMillis();

  ZkStateReader zkr = cloudSolrClient.getZkStateReader();
  zkr.updateClusterState(); // force the state to be fresh

  ClusterState cs = zkr.getClusterState();
  Collection<Slice> slices = cs.getActiveSlices(testCollectionName);
  assertTrue(slices.size() == shards);
  boolean allReplicasUp = false;
  long waitMs = 0L;
  long maxWaitMs = maxWaitSecs * 1000L;
  Replica leader = null;
  while (waitMs < maxWaitMs && !allReplicasUp) {
    // refresh state every 2 secs
    if (waitMs % 2000 == 0) {
      log.info("Updating ClusterState");
      cloudSolrClient.getZkStateReader().updateClusterState();
    }

    cs = cloudSolrClient.getZkStateReader().getClusterState();
    assertNotNull(cs);
    allReplicasUp = true; // assume true
    for (Slice shard : cs.getActiveSlices(testCollectionName)) {
      String shardId = shard.getName();
      assertNotNull("No Slice for " + shardId, shard);
      Collection<Replica> replicas = shard.getReplicas();
      assertTrue(replicas.size() == rf);
      leader = shard.getLeader();
      assertNotNull(leader);
      log.info("Found " + replicas.size() + " replicas and leader on " +
          leader.getNodeName() + " for " + shardId + " in " + testCollectionName);

      // ensure all replicas are "active"
      for (Replica replica : replicas) {
        String replicaState = replica.getStr(ZkStateReader.STATE_PROP);
        if (!"active".equals(replicaState)) {
          log.info("Replica " + replica.getName() + " for shard " + shardId + " is currently " + replicaState);
          allReplicasUp = false;
        }
      }
    }

    if (!allReplicasUp) {
      try {
        Thread.sleep(500L);
      } catch (Exception ignoreMe) {
      }
      waitMs += 500L;
    }
  } // end while

  if (!allReplicasUp)
    fail("Didn't see all replicas for " + testCollectionName +
        " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo(testCollectionName));

  long diffMs = (System.currentTimeMillis() - startMs);
  log.info("Took " + diffMs + " ms to see all replicas become active for " + testCollectionName);
}
 

/**
 * Walks the NamedList response after performing an update request looking for
 * the replication factor that was achieved in each shard involved in the request.
 * For single doc updates, there will be only one shard in the return value.
 */
@SuppressWarnings({"unchecked", "rawtypes"})
public Map<String,Integer> getShardReplicationFactor(String collection, NamedList resp) {
  connect();

  Map<String,Integer> results = new HashMap<String,Integer>();
  if (resp instanceof RouteResponse) {
    NamedList routes = ((RouteResponse)resp).getRouteResponses();
    DocCollection coll = getDocCollection(collection, null);
    Map<String,String> leaders = new HashMap<String,String>();
    for (Slice slice : coll.getActiveSlicesArr()) {
      Replica leader = slice.getLeader();
      if (leader != null) {
        ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
        String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
        leaders.put(leaderUrl, slice.getName());
        String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
        leaders.put(altLeaderUrl, slice.getName());
      }
    }

    @SuppressWarnings({"unchecked"})
    Iterator<Map.Entry<String,Object>> routeIter = routes.iterator();
    while (routeIter.hasNext()) {
      Map.Entry<String,Object> next = routeIter.next();
      String host = next.getKey();
      NamedList hostResp = (NamedList)next.getValue();
      Integer rf = (Integer)((NamedList)hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
      if (rf != null) {
        String shard = leaders.get(host);
        if (shard == null) {
          if (host.endsWith("/"))
            shard = leaders.get(host.substring(0,host.length()-1));
          if (shard == null) {
            shard = host;
          }
        }
        results.put(shard, rf);
      }
    }
  }
  return results;
}