Java Code Examples for org.apache.solr.common.cloud.ClusterState#getLiveNodes()

private void getCheckpoints() throws IOException {
  this.checkpoints = new HashMap<>();
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

  Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();

  for(Slice slice : slices) {
    String sliceName = slice.getName();
    long checkpoint;
    if(initialCheckpoint > -1) {
      checkpoint = initialCheckpoint;
    } else {
      checkpoint = getCheckpoint(slice, liveNodes);
    }

    this.checkpoints.put(sliceName, checkpoint);
  }
}
 

protected static List<String> getActiveReplicaCoreUrls(ZkController zkController, String collection, String localCoreNodeName) {
  List<String> activeReplicaCoreUrls = new ArrayList<>();
  ZkStateReader zkStateReader = zkController.getZkStateReader();
  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null && docCollection.getActiveSlicesArr().length > 0) {
    final Slice[] activeSlices = docCollection.getActiveSlicesArr();
    for (Slice next : activeSlices) {
      Map<String, Replica> replicasMap = next.getReplicasMap();
      if (replicasMap != null) {
        for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
          Replica replica = entry.getValue();
          if (!localCoreNodeName.equals(replica.getName()) &&
              replica.getState() == Replica.State.ACTIVE &&
              liveNodes.contains(replica.getNodeName())) {
            ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(replica);
            activeReplicaCoreUrls.add(replicaCoreProps.getCoreUrl());
          }
        }
      }
    }
  }
  return activeReplicaCoreUrls;
}
 

/**
 * Prepare collection and node / host names for redaction.
 * @param clusterState cluster state
 */
public static RedactionUtils.RedactionContext getRedactionContext(ClusterState clusterState) {
  RedactionUtils.RedactionContext ctx = new RedactionUtils.RedactionContext();
  TreeSet<String> names = new TreeSet<>(clusterState.getLiveNodes());
  for (String nodeName : names) {
    String urlString = Utils.getBaseUrlForNodeName(nodeName, "http");
    try {
      URL u = new URL(urlString);
      // protocol format
      String hostPort = u.getHost() + ":" + u.getPort();
      ctx.addName(u.getHost() + ":" + u.getPort(), RedactionUtils.NODE_REDACTION_PREFIX);
      // node name format
      ctx.addEquivalentName(hostPort, u.getHost() + "_" + u.getPort() + "_", RedactionUtils.NODE_REDACTION_PREFIX);
    } catch (MalformedURLException e) {
      log.warn("Invalid URL for node name {}, replacing including protocol and path", nodeName, e);
      ctx.addName(urlString, RedactionUtils.NODE_REDACTION_PREFIX);
      ctx.addEquivalentName(urlString, Utils.getBaseUrlForNodeName(nodeName, "https"), RedactionUtils.NODE_REDACTION_PREFIX);
    }
  }
  names.clear();
  names.addAll(clusterState.getCollectionStates().keySet());
  names.forEach(n -> ctx.addName(n, RedactionUtils.COLL_REDACTION_PREFIX));
  return ctx;
}
 

public static List<String> getActiveReplicaCoreUrls(ZkController zkController,
                                                    String collection) {
  List<String> activeReplicaCoreUrls = new ArrayList<>();
  ClusterState clusterState = zkController.getZkStateReader().getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null && docCollection.getActiveSlices() != null && docCollection.getActiveSlices().size() > 0) {
    final Collection<Slice> activeSlices = docCollection.getActiveSlices();
    for (Slice next : activeSlices) {
      Map<String, Replica> replicasMap = next.getReplicasMap();
      if (replicasMap != null) {
        for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
          Replica replica = entry.getValue();
          if (replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
            activeReplicaCoreUrls.add(replica.getCoreUrl());
          }
        }
      }
    }
  }
  return activeReplicaCoreUrls;
}
 

protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
  ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection collection = clusterState.getCollectionOrNull(collectionName, true);
  if (collection == null) {
    return null;
  }

  Set<String> liveNodes = clusterState.getLiveNodes();

  if (isPreferLeader) {
    List<Replica> leaderReplicas = collection.getLeaderReplicas(cores.getZkController().getNodeName());
    SolrCore core = randomlyGetSolrCore(liveNodes, leaderReplicas);
    if (core != null) return core;
  }

  List<Replica> replicas = collection.getReplicas(cores.getZkController().getNodeName());
  return randomlyGetSolrCore(liveNodes, replicas);
}
 

private void checkLeaderStatus() throws InterruptedException, KeeperException {
  for (int idx = 0; pendingOps.size() > 0 && idx < 600; ++idx) {
    ClusterState clusterState = coreContainer.getZkController().getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();
    DocCollection dc = clusterState.getCollection(collectionName);
    for (Slice slice : dc.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        if (replica.isActive(liveNodes) && replica.getBool(SliceMutator.PREFERRED_LEADER_PROP, false)) {
          if (replica.getBool(LEADER_PROP, false)) {
            if (pendingOps.containsKey(slice.getName())) {
              // Record for return that the leader changed successfully
              pendingOps.remove(slice.getName());
              addToSuccesses(slice, replica);
              break;
            }
          }
        }
      }
    }
    TimeUnit.MILLISECONDS.sleep(100);
    coreContainer.getZkController().getZkStateReader().forciblyRefreshAllClusterStateSlow();
  }
  addAnyFailures();
}
 

protected List<String> getShardUrls() throws IOException {
  try {
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

    Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    List<String> baseUrls = new ArrayList<>();
    for(Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
          shuffler.add(replica);
        }
      }

      Collections.shuffle(shuffler, new Random());
      Replica rep = shuffler.get(0);
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      baseUrls.add(url);
    }

    return baseUrls;
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

CoreSorter init(CoreContainer cc, Collection<CoreDescriptor> coreDescriptors) {
  String myNodeName = cc.getNodeConfig().getNodeName();
  ClusterState state = cc.getZkController().getClusterState();
  for (CoreDescriptor coreDescriptor : coreDescriptors) {
    CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
    String coll = cloudDescriptor.getCollectionName();
    String sliceName = getShardName(cloudDescriptor);
    if (shardsVsReplicaCounts.containsKey(sliceName)) continue;
    CountsForEachShard c = new CountsForEachShard(0, 0, 0);
    for (Replica replica : getReplicas(state, coll, cloudDescriptor.getShardId())) {
      if (replica.getNodeName().equals(myNodeName)) {
        c.myReplicas++;
      } else {
        Set<String> liveNodes = state.getLiveNodes();
        if (liveNodes.contains(replica.getNodeName())) {
          c.totalReplicasInLiveNodes++;
        } else {
          c.totalReplicasInDownNodes++;
        }
      }
    }
    shardsVsReplicaCounts.put(sliceName, c);
  }

  return this;

}
 

private static List<String> checkLiveNodes(List<String> createNodeList, ClusterState clusterState) {
  Set<String> liveNodes = clusterState.getLiveNodes();
  if (createNodeList != null) {
    if (!liveNodes.containsAll(createNodeList)) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "At least one of the node(s) specified " + createNodeList + " are not currently active in "
              + createNodeList + ", no action taken.");
    }
    // the logic that was extracted to this method used to create a defensive copy but no code
    // was modifying the copy, if this method is made protected or public we want to go back to that
  }
  return createNodeList; // unmodified, but return for inline use
}
 

static HashMap<String, ReplicaCount> getNodeNameVsShardCount(String collectionName,
                                                                     ClusterState clusterState, List<String> createNodeList) {
  HashMap<String, ReplicaCount> nodeNameVsShardCount = new HashMap<>();
  List<String> liveNodes = createNodeList == null || createNodeList.isEmpty() ?
      new ArrayList<>(clusterState.getLiveNodes()) :
      checkLiveNodes(createNodeList, clusterState);

  for (String s : liveNodes) {
    nodeNameVsShardCount.put(s, new ReplicaCount(s));
  }

  // if we were given a list, just use that, don't worry about counts
  if (createNodeList != null) { // Overrides petty considerations about maxShardsPerNode
    return nodeNameVsShardCount;
  }

  // if we get here we were not given a createNodeList, build a map with real counts.
  DocCollection coll = clusterState.getCollection(collectionName);
  int maxShardsPerNode = coll.getMaxShardsPerNode() == -1 ? Integer.MAX_VALUE : coll.getMaxShardsPerNode();
  Map<String, DocCollection> collections = clusterState.getCollectionsMap();
  for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
    DocCollection c = entry.getValue();
    //identify suitable nodes  by checking the no:of cores in each of them
    for (Slice slice : c.getSlices()) {
      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        ReplicaCount count = nodeNameVsShardCount.get(replica.getNodeName());
        if (count != null) {
          count.totalNodes++; // Used to "weigh" whether this node should be used later.
          if (entry.getKey().equals(collectionName)) {
            count.thisCollectionNodes++;
            if (count.thisCollectionNodes >= maxShardsPerNode) nodeNameVsShardCount.remove(replica.getNodeName());
          }
        }
      }
    }
  }

  return nodeNameVsShardCount;
}
 

private String getCoreUrl(String collectionName,
                          String origCorename, ClusterState clusterState, List<Slice> slices,
                          boolean byCoreName, boolean activeReplicas) {
  String coreUrl;
  Set<String> liveNodes = clusterState.getLiveNodes();
  Collections.shuffle(slices, random);

  for (Slice slice : slices) {
    List<Replica> randomizedReplicas = new ArrayList<>(slice.getReplicas());
    Collections.shuffle(randomizedReplicas, random);

    for (Replica replica : randomizedReplicas) {
      if (!activeReplicas || (liveNodes.contains(replica.getNodeName())
          && replica.getState() == Replica.State.ACTIVE)) {

        if (byCoreName && !origCorename.equals(replica.getStr(CORE_NAME_PROP))) {
          // if it's by core name, make sure they match
          continue;
        }
        if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
          // don't count a local core
          continue;
        }

        if (origCorename != null) {
          coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
        } else {
          coreUrl = replica.getCoreUrl();
          if (coreUrl.endsWith("/")) {
            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
          }
        }

        return coreUrl;
      }
    }
  }
  return null;
}
 

/**
 * Pick one replica from each shard to send the shard requests to.
 *
 * @param collection that is being queried
 * @throws IOException if an exception occurs while finding replicas
 */
protected void pickShards(String collection) throws IOException {
  try {

    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    Slice[] slices = clusterState.getCollection(collection).getActiveSlicesArr();

    for(Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName()))
        shuffler.add(replica);
      }

      Collections.shuffle(shuffler, new Random());
      Replica rep = shuffler.get(0);
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      replicaUrls.add(url);
    }
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

private List<String> getShardUrls() throws IOException {
  try {
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

    Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    List<String> baseUrls = new ArrayList<>();
    for(Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
          shuffler.add(replica);
        }
      }

      Collections.shuffle(shuffler, new Random());
      Replica rep = shuffler.get(0);
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      baseUrls.add(url);
    }

    return baseUrls;

  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

private void getPersistedCheckpoints() throws IOException {
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
  Slice[] slices = CloudSolrStream.getSlices(checkpointCollection, zkStateReader, false);

  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();

  OUTER:
  for(Slice slice : slices) {
    Collection<Replica> replicas = slice.getReplicas();
    for(Replica replica : replicas) {
      if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())){
        HttpSolrClient httpClient = streamContext.getSolrClientCache().getHttpSolrClient(replica.getCoreUrl());
        try {
          SolrDocument doc = httpClient.getById(id);
          if(doc != null) {
            @SuppressWarnings({"unchecked"})
            List<String> checkpoints = (List<String>)doc.getFieldValue("checkpoint_ss");
            for (String checkpoint : checkpoints) {
              String[] pair = checkpoint.split("~");
              this.checkpoints.put(pair[0], Long.parseLong(pair[1]));
            }
          }
        } catch (Exception e) {
          throw new IOException(e);
        }
        break OUTER;
      }
    }
  }
}
 

@Override
public Set<String> getLiveNodes() {
  ClusterState clusterState = getZkStateReader().getClusterState();
  if (clusterState != null) {
    return clusterState.getLiveNodes();
  } else {
    return Collections.emptySet();
  }
}
 

@SuppressWarnings({"unchecked"})
public static List<String> getShards(String zkHost,
                                     String collection,
                                     StreamContext streamContext,
                                     SolrParams requestParams)
    throws IOException {
  Map<String, List<String>> shardsMap = null;
  List<String> shards = new ArrayList<>();

  if(streamContext != null) {
    shardsMap = (Map<String, List<String>>)streamContext.get("shards");
  }

  if(shardsMap != null) {
    //Manual Sharding
    shards = shardsMap.get(collection);
  } else {
    //SolrCloud Sharding
    CloudSolrClient cloudSolrClient =
        Optional.ofNullable(streamContext.getSolrClientCache()).orElseGet(SolrClientCache::new).getCloudSolrClient(zkHost);
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    Slice[] slices = CloudSolrStream.getSlices(collection, zkStateReader, true);
    Set<String> liveNodes = clusterState.getLiveNodes();


    ModifiableSolrParams solrParams = new ModifiableSolrParams(streamContext.getRequestParams());
    solrParams.add(requestParams);

    RequestReplicaListTransformerGenerator requestReplicaListTransformerGenerator =
        Optional.ofNullable(streamContext.getRequestReplicaListTransformerGenerator()).orElseGet(RequestReplicaListTransformerGenerator::new);

    ReplicaListTransformer replicaListTransformer = requestReplicaListTransformerGenerator.getReplicaListTransformer(solrParams);

    for(Slice slice : slices) {
      List<Replica> sortedReplicas = new ArrayList<>();
      for(Replica replica : slice.getReplicas()) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
          sortedReplicas.add(replica);
        }
      }

      replicaListTransformer.transform(sortedReplicas);
      if (sortedReplicas.size() > 0) {
        shards.add(sortedReplicas.get(0).getCoreUrl());
      }
    }
  }
  Object core = streamContext.get("core");
  if (streamContext != null && streamContext.isLocal() && core != null) {
    shards.removeIf(shardUrl -> !shardUrl.contains((CharSequence) core));
  }

  return shards;
}
 

protected void constructStreams() throws IOException {
  try {
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
    Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

    ModifiableSolrParams mParams = new ModifiableSolrParams(params);
    mParams.set(DISTRIB, "false"); // We are the aggregator.
    String fl = mParams.get("fl");
    mParams.set(SORT, "_version_ asc");
    if(!fl.contains(VERSION_FIELD)) {
      fl += ",_version_";
    }
    mParams.set("fl", fl);

    Random random = new Random();

    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    for(Slice slice : slices) {
      ModifiableSolrParams localParams = new ModifiableSolrParams(mParams);
      long checkpoint = checkpoints.get(slice.getName());

      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName()))
          shuffler.add(replica);
      }

      Replica rep = shuffler.get(random.nextInt(shuffler.size()));
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      SolrStream solrStream = new SolrStream(url, localParams);
      solrStream.setSlice(slice.getName());
      solrStream.setCheckpoint(checkpoint);
      solrStream.setTrace(true);
      if(streamContext != null) {
        solrStream.setStreamContext(streamContext);
      }
      solrStreams.add(solrStream);
    }
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

@Test
public void testCoreRegistration() throws Exception {
  System.setProperty("solrcloud.update.delay", "1");

  assertEquals(0, CollectionAdminRequest.createCollection("testcore", "conf", 1, 1)
      .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName())
      .process(cluster.getSolrClient()).getStatus());

  ZkController zkController2 = cluster.getJettySolrRunner(1).getCoreContainer().getZkController();

  String host = zkController2.getHostName();
  
  // slight pause - TODO: takes an oddly long amount of time to schedule tasks
  // with almost no delay ...
  ClusterState clusterState2 = null;
  Map<String,Slice> slices = null;
  for (int i = 75; i > 0; i--) {
    clusterState2 = zkController2.getClusterState();
    DocCollection docCollection = clusterState2.getCollectionOrNull("testcore");
    slices = docCollection == null ? null : docCollection.getSlicesMap();
    
    if (slices != null && slices.containsKey("shard1")
        && slices.get("shard1").getReplicasMap().size() > 0) {
      break;
    }
    Thread.sleep(500);
  }

  assertNotNull(slices);
  assertTrue(slices.containsKey("shard1"));

  Slice slice = slices.get("shard1");
  assertEquals("shard1", slice.getName());

  Map<String,Replica> shards = slice.getReplicasMap();

  assertEquals(1, shards.size());

  // assert this is core of container1
  Replica zkProps = shards.values().iterator().next();

  assertNotNull(zkProps);

  assertEquals(host + ":" +cluster.getJettySolrRunner(0).getLocalPort()+"_solr", zkProps.getStr(ZkStateReader.NODE_NAME_PROP));

  assertTrue(zkProps.getStr(ZkStateReader.BASE_URL_PROP).contains("http://" + host + ":"+cluster.getJettySolrRunner(0).getLocalPort()+"/solr")
    || zkProps.getStr(ZkStateReader.BASE_URL_PROP).contains("https://" + host + ":"+cluster.getJettySolrRunner(0).getLocalPort()+"/solr") );

  // assert there are 3 live nodes
  Set<String> liveNodes = clusterState2.getLiveNodes();
  assertNotNull(liveNodes);
  assertEquals(3, liveNodes.size());

  // shut down node 2
  JettySolrRunner j = cluster.stopJettySolrRunner(2);

  // slight pause (15s timeout) for watch to trigger
  for(int i = 0; i < (5 * 15); i++) {
    if(zkController2.getClusterState().getLiveNodes().size() == 2) {
      break;
    }
    Thread.sleep(200);
  }
  
  cluster.waitForJettyToStop(j);

  assertEquals(2, zkController2.getClusterState().getLiveNodes().size());

  cluster.getJettySolrRunner(1).stop();
  cluster.getJettySolrRunner(1).start();
  
  // pause for watch to trigger
  for(int i = 0; i < 200; i++) {
    if (cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getClusterState().liveNodesContain(
        cluster.getJettySolrRunner(1).getCoreContainer().getZkController().getNodeName())) {
      break;
    }
    Thread.sleep(100);
  }

  assertTrue(cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getClusterState().liveNodesContain(
      cluster.getJettySolrRunner(1).getCoreContainer().getZkController().getNodeName()));

  // core.close();  // don't close - this core is managed by container1 now
}
 

@Test
public void test() throws Exception {
  CloudSolrClient cloudClient = cluster.getSolrClient();
  String coll = "deletenodetest_coll";
  ClusterState state = cloudClient.getZkStateReader().getClusterState();
  Set<String> liveNodes = state.getLiveNodes();
  ArrayList<String> l = new ArrayList<>(liveNodes);
  Collections.shuffle(l, random());
  CollectionAdminRequest.Create create = pickRandom(
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1),
      // check RF=1
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 0)
      );
  create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3);
  cloudClient.request(create);
  state = cloudClient.getZkStateReader().getClusterState();
  String node2bdecommissioned = l.get(0);
  // check what replicas are on the node, and whether the call should fail
  boolean shouldFail = false;
  DocCollection docColl = state.getCollection(coll);
  log.info("#### DocCollection: {}", docColl);
  List<Replica> replicas = docColl.getReplicas(node2bdecommissioned);
  if (replicas != null) {
    for (Replica replica : replicas) {
      String shard = docColl.getShardId(node2bdecommissioned, replica.getStr(ZkStateReader.CORE_NAME_PROP));
      Slice slice = docColl.getSlice(shard);
      boolean hasOtherNonPullReplicas = false;
      for (Replica r: slice.getReplicas()) {
        if (!r.getName().equals(replica.getName()) &&
            !r.getNodeName().equals(node2bdecommissioned) &&
            r.getType() != Replica.Type.PULL) {
          hasOtherNonPullReplicas = true;
          break;
        }
      }
      if (!hasOtherNonPullReplicas) {
        shouldFail = true;
        break;
      }
    }
  }
  new CollectionAdminRequest.DeleteNode(node2bdecommissioned).processAsync("003", cloudClient);
  CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("003");
  CollectionAdminRequest.RequestStatusResponse rsp = null;
  for (int i = 0; i < 200; i++) {
    rsp = requestStatus.process(cloudClient);
    if (rsp.getRequestStatus() == RequestStatusState.FAILED || rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
      break;
    }
    Thread.sleep(50);
  }
  if (log.isInfoEnabled()) {
    log.info("####### DocCollection after: {}", cloudClient.getZkStateReader().getClusterState().getCollection(coll));
  }
  if (shouldFail) {
    assertTrue(String.valueOf(rsp), rsp.getRequestStatus() == RequestStatusState.FAILED);
  } else {
    assertFalse(String.valueOf(rsp), rsp.getRequestStatus() == RequestStatusState.FAILED);
  }
}