Java Code Examples for org.apache.solr.common.cloud.ClusterState#getCollectionOrNull()

public static void verifyPropertyNotPresent(CloudSolrClient client, String collectionName, String replicaName,
                              String property)
    throws KeeperException, InterruptedException {
  ClusterState clusterState = null;
  Replica replica = null;
  for (int idx = 0; idx < 300; ++idx) {
    clusterState = client.getZkStateReader().getClusterState();
    final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName);
    replica = (docCollection == null) ? null : docCollection.getReplica(replicaName);
    if (replica == null) {
      fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
    }
    if (StringUtils.isBlank(replica.getProperty(property))) return;
    Thread.sleep(100);
  }
  fail("Property " + property + " not set correctly for collection/replica pair: " +
      collectionName + "/" + replicaName + ". Replica props: " + replica.getProperties().toString() +
      ". Cluster state is " + clusterState.toString());

}
 

private Map<String, String> getProps(CloudSolrClient client, String collectionName, String replicaName, String... props)
    throws KeeperException, InterruptedException {

  client.getZkStateReader().forceUpdateCollection(collectionName);
  ClusterState clusterState = client.getZkStateReader().getClusterState();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName);
  if (docCollection == null || docCollection.getReplica(replicaName) == null) {
    fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
  }
  Replica replica = docCollection.getReplica(replicaName);
  Map<String, String> propMap = new HashMap<>();
  for (String prop : props) {
    propMap.put(prop, replica.getProperty(prop));
  }
  return propMap;
}
 

protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
  ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection collection = clusterState.getCollectionOrNull(collectionName, true);
  if (collection == null) {
    return null;
  }

  Set<String> liveNodes = clusterState.getLiveNodes();

  if (isPreferLeader) {
    List<Replica> leaderReplicas = collection.getLeaderReplicas(cores.getZkController().getNodeName());
    SolrCore core = randomlyGetSolrCore(liveNodes, leaderReplicas);
    if (core != null) return core;
  }

  List<Replica> replicas = collection.getReplicas(cores.getZkController().getNodeName());
  return randomlyGetSolrCore(liveNodes, replicas);
}
 

public static List<String> getActiveReplicaCoreUrls(ZkController zkController,
                                                    String collection) {
  List<String> activeReplicaCoreUrls = new ArrayList<>();
  ClusterState clusterState = zkController.getZkStateReader().getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null && docCollection.getActiveSlices() != null && docCollection.getActiveSlices().size() > 0) {
    final Collection<Slice> activeSlices = docCollection.getActiveSlices();
    for (Slice next : activeSlices) {
      Map<String, Replica> replicasMap = next.getReplicasMap();
      if (replicasMap != null) {
        for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
          Replica replica = entry.getValue();
          if (replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
            activeReplicaCoreUrls.add(replica.getCoreUrl());
          }
        }
      }
    }
  }
  return activeReplicaCoreUrls;
}
 

@SuppressWarnings("WeakerAccess")
Set<String> getLeaderCoreNames(ClusterState clusterState) {
  Set<String> leaders = new TreeSet<>(); // sorted just to make it easier to read when debugging...
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (JettySolrRunner jettySolrRunner : jettySolrRunners) {
    List<CoreDescriptor> coreDescriptors = jettySolrRunner.getCoreContainer().getCoreDescriptors();
    for (CoreDescriptor core : coreDescriptors) {
      String nodeName = jettySolrRunner.getNodeName();
      String collectionName = core.getCollectionName();
      DocCollection collectionOrNull = clusterState.getCollectionOrNull(collectionName);
      List<Replica> leaderReplicas = collectionOrNull.getLeaderReplicas(nodeName);
      if (leaderReplicas != null) {
        for (Replica leaderReplica : leaderReplicas) {
          leaders.add(leaderReplica.getCoreName());
        }
      }
    }
  }
  return leaders;
}
 

@Test
public void testBuildClusterState_ReplicaStateAndType() {
  try (ZkStateReader zkStateReader = ClusterStateMockUtil.buildClusterState("csrStRpDnF", "baseUrl1_")) {
    ClusterState clusterState = zkStateReader.getClusterState();
    assertNotNull(clusterState);
    assertEquals(1, clusterState.getCollectionStates().size());
    DocCollection collection1 = clusterState.getCollectionOrNull("collection1");
    assertNotNull(collection1);
    assertEquals(DocRouter.DEFAULT, collection1.getRouter());
    assertEquals(1, collection1.getActiveSlices().size());
    assertEquals(1, collection1.getSlices().size());
    Slice slice1 = collection1.getSlice("slice1");
    assertNotNull(slice1);
    assertEquals(4, slice1.getReplicas().size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.NRT && replica.getState() == Replica.State.ACTIVE).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.NRT && replica.getState() == Replica.State.RECOVERY_FAILED).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.TLOG && replica.getState() == Replica.State.RECOVERING).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.PULL && replica.getState() == Replica.State.DOWN).size());
  }
}
 

protected static List<String> getActiveReplicaCoreUrls(ZkController zkController, String collection, String localCoreNodeName) {
  List<String> activeReplicaCoreUrls = new ArrayList<>();
  ZkStateReader zkStateReader = zkController.getZkStateReader();
  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null && docCollection.getActiveSlicesArr().length > 0) {
    final Slice[] activeSlices = docCollection.getActiveSlicesArr();
    for (Slice next : activeSlices) {
      Map<String, Replica> replicasMap = next.getReplicasMap();
      if (replicasMap != null) {
        for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
          Replica replica = entry.getValue();
          if (!localCoreNodeName.equals(replica.getName()) &&
              replica.getState() == Replica.State.ACTIVE &&
              liveNodes.contains(replica.getNodeName())) {
            ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(replica);
            activeReplicaCoreUrls.add(replicaCoreProps.getCoreUrl());
          }
        }
      }
    }
  }
  return activeReplicaCoreUrls;
}
 

/**
 * Handles state updates
 */
public ZkWriteCommand setState(ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);

  if (collectionName == null || sliceName == null) {
    log.error("Invalid collection and slice {}", message);
    return ZkStateWriter.NO_OP;
  }
  DocCollection collection = clusterState.getCollectionOrNull(collectionName);
  Slice slice = collection != null ? collection.getSlice(sliceName) : null;
  if (slice == null) {
    log.error("No such slice exists {}", message);
    return ZkStateWriter.NO_OP;
  }

  return updateState(clusterState, message);
}
 

@Test
public void testBuildClusterState_Simple() {
  try (ZkStateReader zkStateReader = ClusterStateMockUtil.buildClusterState("csr", "baseUrl1_")) {
    ClusterState clusterState = zkStateReader.getClusterState();
    assertNotNull(clusterState);
    assertEquals(1, clusterState.getCollectionStates().size());
    DocCollection collection1 = clusterState.getCollectionOrNull("collection1");
    assertNotNull(collection1);
    assertEquals(DocRouter.DEFAULT, collection1.getRouter());
    assertEquals(1, collection1.getActiveSlices().size());
    assertEquals(1, collection1.getSlices().size());
    Slice slice1 = collection1.getSlice("slice1");
    assertNotNull(slice1);
    assertEquals(1, slice1.getReplicas().size());
    Replica replica1 = slice1.getReplica("replica1");
    assertNotNull(replica1);
    assertEquals("baseUrl1_", replica1.getNodeName());
    assertEquals("slice1_replica1", replica1.getCoreName());
    assertEquals("http://baseUrl1", replica1.getBaseUrl());
    assertEquals("http://baseUrl1/slice1_replica1/", replica1.getCoreUrl());
    assertEquals(Replica.State.ACTIVE, replica1.getState());
    assertEquals(Replica.Type.NRT, replica1.getType());
  }
}
 

/**
 * Checks if the collection has already been created in Solr.
 */
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    zkStateReader.updateClusterState();
    ClusterState clusterState = zkStateReader.getClusterState();
    return clusterState.getCollectionOrNull(collection) != null;
}
 

/**
 * Checks if the collection has already been created in Solr.
 */
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    zkStateReader.updateClusterState(true);
    ClusterState clusterState = zkStateReader.getClusterState();
    return clusterState.getCollectionOrNull(collection) != null;
}
 

private Replica getSystemCollReplica() {
  ZkStateReader zkStateReader = this.coreContainer.getZkController().getZkStateReader();
  ClusterState cs = zkStateReader.getClusterState();
  DocCollection coll = cs.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);
  if (coll == null)
    throw new SolrException(SERVICE_UNAVAILABLE, CollectionAdminParams.SYSTEM_COLL + " collection not available");
  ArrayList<Slice> slices = new ArrayList<>(coll.getActiveSlices());
  if (slices.isEmpty())
    throw new SolrException(SERVICE_UNAVAILABLE, "No active slices for " + CollectionAdminParams.SYSTEM_COLL + " collection");
  Collections.shuffle(slices, RANDOM); //do load balancing

  Replica replica = null;
  for (Slice slice : slices) {
    List<Replica> replicas = new ArrayList<>(slice.getReplicasMap().values());
    Collections.shuffle(replicas, RANDOM);
    for (Replica r : replicas) {
      if (r.getState() == Replica.State.ACTIVE) {
        if (zkStateReader.getClusterState().getLiveNodes().contains(r.get(ZkStateReader.NODE_NAME_PROP))) {
          replica = r;
          break;
        } else {
          if (log.isInfoEnabled()) {
            log.info("replica {} says it is active but not a member of live nodes", r.get(ZkStateReader.NODE_NAME_PROP));
          }
        }
      }
    }
  }
  if (replica == null) {
    throw new SolrException(SERVICE_UNAVAILABLE, "No active replica available for " + CollectionAdminParams.SYSTEM_COLL + " collection");
  }
  return replica;
}
 

/**Return all replicas for a given collection+slice combo
 */
private Collection<Replica> getReplicas(ClusterState cs, String coll, String slice) {
  DocCollection c = cs.getCollectionOrNull(coll);
  if (c == null) return emptyList();
  Slice s = c.getSlice(slice);
  if (s == null) return emptyList();
  return s.getReplicas();
}
 

/**
 * Checks if the collection has already been created in Solr.
 */
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    final ZkStateReader zkStateReader = server.getZkStateReader();
    zkStateReader.forceUpdateCollection(collection);
    final ClusterState clusterState = zkStateReader.getClusterState();
    return clusterState.getCollectionOrNull(collection) != null;
}
 

/**
 * Wait for a particular collection state to appear.
 *
 * This is a convenience method using the {@link #DEFAULT_TIMEOUT}
 *
 * @param cloudManager current instance of {@link SolrCloudManager}
 * @param collection  the collection to watch
 * @param wait timeout value
 * @param unit timeout unit
 * @param predicate   a predicate to match against the collection state
 */
public static long waitForState(final SolrCloudManager cloudManager,
                                final String collection,
                                long wait,
                                final TimeUnit unit,
                                final CollectionStatePredicate predicate) throws InterruptedException, TimeoutException, IOException {
  TimeOut timeout = new TimeOut(wait, unit, cloudManager.getTimeSource());
  long timeWarn = timeout.timeLeft(TimeUnit.MILLISECONDS) / 4;
  ClusterState state = null;
  DocCollection coll = null;
  while (!timeout.hasTimedOut()) {
    state = cloudManager.getClusterStateProvider().getClusterState();
    coll = state.getCollectionOrNull(collection);
    // due to the way we manage collections in SimClusterStateProvider a null here
    // can mean that a collection is still being created but has no replicas
    if (coll == null) { // does not yet exist?
      timeout.sleep(100);
      continue;
    }
    if (predicate.matches(state.getLiveNodes(), coll)) {
      log.trace("-- predicate matched with state {}", state);
      return timeout.timeElapsed(TimeUnit.MILLISECONDS);
    }
    timeout.sleep(100);
    if (timeout.timeLeft(TimeUnit.MILLISECONDS) < timeWarn) {
      log.trace("-- still not matching predicate: {}", state);
    }
  }
  throw new TimeoutException("last ClusterState: " + state + ", last coll state: " + coll);
}
 

public static boolean replicaExists(ClusterState clusterState, String collection, String shard, String coreNodeName) {
  DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null) {
    Slice slice = docCollection.getSlice(shard);
    if (slice != null) {
      return slice.getReplica(coreNodeName) != null;
    }
  }
  return false;
}
 

@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context,
             Throwable error, String message) throws Exception {
  try {
    ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
    DocCollection coll = clusterState.getCollectionOrNull(collection);
    if (coll == null) {
      log.debug("Collection {} missing, skip sending event {}", collection, event);
      return;
    }
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField(CommonParams.TYPE, DOC_TYPE);
    doc.addField(SOURCE_FIELD, SOURCE);
    doc.addField("id", IdUtils.timeRandomId());
    doc.addField("event.id_s", event.getId());
    doc.addField(EVENT_TYPE_FIELD, event.getEventType().toString());
    doc.addField(EVENT_SOURCE_FIELD, event.getSource());
    doc.addField("event.time_l", event.getEventTime());
    doc.addField("timestamp", new Date());
    addMap("event.property.", doc, event.getProperties());
    doc.addField(STAGE_FIELD, stage.toString());
    if (actionName != null) {
      doc.addField(ACTION_FIELD, actionName);
    }
    if (message != null) {
      doc.addField(MESSAGE_FIELD, message);
    }
    addError(doc, error);
    // add JSON versions of event and context
    String eventJson = Utils.toJSONString(event);
    doc.addField("event_str", eventJson);
    if (context != null) {
      // capture specifics of operations after compute_plan action
      addOperations(doc, (List<SolrRequest>)context.getProperties().get("operations"));
      // capture specifics of responses after execute_plan action
      addResponses(doc, (List<NamedList<Object>>)context.getProperties().get("responses"));
      addActions(BEFORE_ACTIONS_FIELD, doc, (List<String>)context.getProperties().get(TriggerEventProcessorStage.BEFORE_ACTION.toString()));
      addActions(AFTER_ACTIONS_FIELD, doc, (List<String>)context.getProperties().get(TriggerEventProcessorStage.AFTER_ACTION.toString()));
      String contextJson = Utils.toJSONString(context);
      doc.addField("context_str", contextJson);
    }
    UpdateRequest req = new UpdateRequest();
    req.add(doc);
    req.setParam(CollectionAdminParams.COLLECTION, collection);
    cloudManager.request(req);
  } catch (Exception e) {
    if ((e instanceof SolrException) && e.getMessage().contains("Collection not found")) {
      // relatively benign but log this - collection still existed when we started
      log.info("Collection {} missing, skip sending event {}", collection, event);
    } else {
      log.warn("Exception sending event. Collection: {}, event: {}, exception: {}", collection, event, e);
    }
  }
}
 

private Replica getReplica(ClusterState clusterState, String collectionName, String replicaName) {
  if (clusterState == null) return null;
  final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName);
  if (docCollection == null) return null;
  return docCollection.getReplica(replicaName);
}
 

protected void waitToSeeReplicasActive(String testCollectionName, String shardId, Set<String> replicasToCheck, int maxWaitSecs) throws Exception {
  final RTimer timer = new RTimer();

  ZkStateReader zkr = cloudClient.getZkStateReader();
  zkr.forceUpdateCollection(testCollectionName);
  ClusterState cs = zkr.getClusterState();
  boolean allReplicasUp = false;
  long waitMs = 0L;
  long maxWaitMs = maxWaitSecs * 1000L;
  while (waitMs < maxWaitMs && !allReplicasUp) {
    cs = cloudClient.getZkStateReader().getClusterState();
    assertNotNull(cs);
    final DocCollection docCollection = cs.getCollectionOrNull(testCollectionName);
    assertNotNull(docCollection);
    Slice shard = docCollection.getSlice(shardId);
    assertNotNull("No Slice for "+shardId, shard);
    allReplicasUp = true; // assume true

    // wait to see all replicas are "active"
    for (Replica replica : shard.getReplicas()) {
      if (!replicasToCheck.contains(replica.getName()))
        continue;

      final Replica.State state = replica.getState();
      if (state != Replica.State.ACTIVE) {
        if (log.isInfoEnabled()) {
          log.info("Replica {} is currently {}", replica.getName(), state);
        }
        allReplicasUp = false;
      }
    }

    if (!allReplicasUp) {
      try {
        Thread.sleep(200L);
      } catch (Exception ignoreMe) {}
      waitMs += 200L;
    }
  } // end while

  if (!allReplicasUp)
    fail("Didn't see replicas "+ replicasToCheck +
        " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo(testCollectionName));

  if (log.isInfoEnabled()) {
    log.info("Took {} ms to see replicas [{}] become active.", timer.getTime(), replicasToCheck);
  }
}
 

@Override
public DocCollection getCollection(String name) throws IOException {
  ClusterState cs = getClusterState();
  return cs == null ? null : cs.getCollectionOrNull(name);
}