Java Code Examples for org.apache.solr.common.cloud.ClusterState#forEachCollection()

protected int getMaxNumOps(TriggerEvent event, AutoScalingConfig autoScalingConfig, ClusterState clusterState) {
  // estimate a maximum default limit that should be sufficient for most purposes:
  // number of nodes * total number of replicas * 3
  AtomicInteger totalRF = new AtomicInteger();
  clusterState.forEachCollection(coll -> {
    Integer rf = coll.getReplicationFactor();
    if (rf == null) {
      if (coll.getSlices().isEmpty()) {
        rf = 1; // ???
      } else {
        rf = coll.getReplicas().size() / coll.getSlices().size();
      }
    }
    totalRF.addAndGet(rf * coll.getSlices().size());
  });
  int totalMax = clusterState.getLiveNodes().size() * totalRF.get() * 3;
  int maxOp = (Integer) autoScalingConfig.getProperties().getOrDefault(AutoScalingParams.MAX_COMPUTE_OPERATIONS, totalMax);
  Object o = event.getProperty(AutoScalingParams.MAX_COMPUTE_OPERATIONS, maxOp);
  try {
    return Integer.parseInt(String.valueOf(o));
  } catch (Exception e) {
    log.warn("Invalid '{}' event property: {}, using default {}", AutoScalingParams.MAX_COMPUTE_OPERATIONS, o, maxOp);
    return maxOp;
  }
}
 

private static void assertClusterStateEquals(ClusterState one, ClusterState two) {
  assertEquals(one.getLiveNodes(), two.getLiveNodes());
  assertEquals(one.getCollectionsMap().keySet(), two.getCollectionsMap().keySet());
  one.forEachCollection(oneColl -> {
    DocCollection twoColl = two.getCollection(oneColl.getName());
    Map<String, Slice> oneSlices = oneColl.getSlicesMap();
    Map<String, Slice> twoSlices = twoColl.getSlicesMap();
    assertEquals(oneSlices.keySet(), twoSlices.keySet());
    oneSlices.forEach((s, slice) -> {
      Slice sTwo = twoSlices.get(s);
      for (Replica oneReplica : slice.getReplicas()) {
        Replica twoReplica = sTwo.getReplica(oneReplica.getName());
        assertNotNull(twoReplica);
        SimSolrCloudTestCase.assertReplicaEquals(oneReplica, twoReplica);
      }
    });
  });
}
 

/**
 * Compare two ClusterState-s, filtering out simulation framework artifacts.
 */
public static void assertClusterStateEquals(ClusterState one, ClusterState two) {
  assertEquals(one.getLiveNodes(), two.getLiveNodes());
  assertEquals(one.getCollectionsMap().keySet(), two.getCollectionsMap().keySet());
  one.forEachCollection(oneColl -> {
    DocCollection twoColl = two.getCollection(oneColl.getName());
    Map<String, Slice> oneSlices = oneColl.getSlicesMap();
    Map<String, Slice> twoSlices = twoColl.getSlicesMap();
    assertEquals(oneSlices.keySet(), twoSlices.keySet());
    oneSlices.forEach((s, slice) -> {
      Slice sTwo = twoSlices.get(s);
      for (Replica oneReplica : slice.getReplicas()) {
        Replica twoReplica = sTwo.getReplica(oneReplica.getName());
        assertNotNull(twoReplica);
        assertReplicaEquals(oneReplica, twoReplica);
      }
    });
  });
}
 

@After
public void printState() throws Exception {
  log.debug("-------------_ FINAL STATE --------------");
  SolrCloudManager cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
  for (String node: cloudManager.getClusterStateProvider().getLiveNodes()) {
    Map<String, Object> values = cloudManager.getNodeStateProvider().getNodeValues(node, ImplicitSnitch.tags);
    if (log.isDebugEnabled()) {
      log.debug("* Node values: {}\n{}", node, Utils.toJSONString(values));
    }
  }
  if (log.isDebugEnabled()) {
    log.debug("* Live nodes: {}", cloudManager.getClusterStateProvider().getLiveNodes());
  }
  ClusterState state = cloudManager.getClusterStateProvider().getClusterState();
  if (log.isDebugEnabled()) {
    state.forEachCollection(coll -> log.debug("* Collection {} state: {}", coll.getName(), coll));
  }
}
 

/**
 * Simulate running a shard leader election. This operation is a no-op if a leader already exists.
 * If a new leader is elected the cluster state is saved.
 * @param collections list of affected collections
 * @param saveClusterState if true then save cluster state regardless of changes.
 */
private void simRunLeaderElection(Collection<String> collections, boolean saveClusterState) throws Exception {
  ensureNotClosed();
  if (saveClusterState) {
    lock.lockInterruptibly();
    try {
      collections.forEach(c -> collectionsStatesRef.get(c).invalidate());
    } finally {
      lock.unlock();
    }
  }
  ClusterState state = getClusterState();
  state.forEachCollection(dc -> {
      if (!collections.contains(dc.getName())) {
        return;
      }
      dc.getSlices().forEach(s -> {
        if (log.isTraceEnabled()) {
          log.trace("-- submit leader election for {} / {}", dc.getName(), s.getName());
        }
          cloudManager.submit(() -> {
              simRunLeaderElection(dc.getName(), s.getName(), saveClusterState);
              return true;
            });
        });
    });
}
 

public String dumpClusterState(boolean withCollections) throws Exception {
  StringBuilder sb = new StringBuilder();
  sb.append("#######################################\n");
  sb.append("############ CLUSTER STATE ############\n");
  sb.append("#######################################\n");
  sb.append("## Live nodes:\t\t").append(getLiveNodesSet().size()).append("\n");
  int emptyNodes = 0;
  int maxReplicas = 0;
  int minReplicas = Integer.MAX_VALUE;
  Map<String, Map<Replica.State, AtomicInteger>> replicaStates = new TreeMap<>();
  int numReplicas = 0;
  for (String node : getLiveNodesSet().get()) {
    List<ReplicaInfo> replicas = getSimClusterStateProvider().simGetReplicaInfos(node);
    numReplicas += replicas.size();
    if (replicas.size() > maxReplicas) {
      maxReplicas = replicas.size();
    }
    if (minReplicas > replicas.size()) {
      minReplicas = replicas.size();
    }
    for (ReplicaInfo ri : replicas) {
      replicaStates.computeIfAbsent(ri.getCollection(), c -> new TreeMap<>())
          .computeIfAbsent(ri.getState(), s -> new AtomicInteger())
          .incrementAndGet();
    }
    if (replicas.isEmpty()) {
      emptyNodes++;
    }
  }
  if (minReplicas == Integer.MAX_VALUE) {
    minReplicas = 0;
  }
  sb.append("## Empty nodes:\t").append(emptyNodes).append("\n");
  Set<String> deadNodes = getSimNodeStateProvider().simGetDeadNodes();
  sb.append("## Dead nodes:\t\t").append(deadNodes.size()).append("\n");
  deadNodes.forEach(n -> sb.append("##\t\t").append(n).append("\n"));
  sb.append("## Collections:\n");
    clusterStateProvider.simGetCollectionStats().forEach((coll, stats) -> {
      sb.append("##  * ").append(coll).append('\n');
      stats.forEach((k, v) -> {
        sb.append("##    ").append(k).append("\t").append(v).append("\n");
      });
    });
  if (withCollections) {
    ClusterState state = clusterStateProvider.getClusterState();
    state.forEachCollection(coll -> sb.append(coll.toString()).append("\n"));
  }
  sb.append("## Max replicas per node:\t").append(maxReplicas).append("\n");
  sb.append("## Min replicas per node:\t").append(minReplicas).append("\n");
  sb.append("## Total replicas:\t\t").append(numReplicas).append("\n");
  replicaStates.forEach((c, map) -> {
    AtomicInteger repCnt = new AtomicInteger();
    map.forEach((s, cnt) -> repCnt.addAndGet(cnt.get()));
    sb.append("## * ").append(c).append("\t\t").append(repCnt.get()).append("\n");
    map.forEach((s, cnt) -> sb.append("##\t\t- ").append(String.format(Locale.ROOT, "%-12s  %4d", s, cnt.get())).append("\n"));
  });
  sb.append("######### Solr op counts ##########\n");
  simGetOpCounts().forEach((k, cnt) -> sb.append("##\t\t- ").append(String.format(Locale.ROOT, "%-14s  %4d", k, cnt.get())).append("\n"));
  sb.append("######### Autoscaling event counts ###########\n");
  Map<String, Map<String, AtomicInteger>> counts = simGetEventCounts();
  counts.forEach((trigger, map) -> {
    sb.append("## * Trigger: ").append(trigger).append("\n");
    map.forEach((s, cnt) -> sb.append("##\t\t- ").append(String.format(Locale.ROOT, "%-11s  %4d", s, cnt.get())).append("\n"));
  });
  return sb.toString();
}