Java Code Examples for org.apache.solr.common.cloud.Replica#getState()

private CollectionStatePredicate clusterStateReflectsActiveAndDownReplicas() {
  return (liveNodes, collectionState) -> {
    for (Replica r:collectionState.getReplicas()) {
      if (r.getState() != Replica.State.DOWN && r.getState() != Replica.State.ACTIVE) {
        return false;
      }
      if (r.getState() == Replica.State.DOWN && liveNodes.contains(r.getNodeName())) {
        return false;
      }
      if (r.getState() == Replica.State.ACTIVE && !liveNodes.contains(r.getNodeName())) {
        return false;
      }
    }
    return true;
  };
}
 

private void waitTillAllNodesActive() throws Exception {
  for (int i = 0; i < 60; i++) { 
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;
    for (Replica replica : replicas) {
      if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see all nodes active");
}
 

private CollectionStatePredicate clusterStateReflectsActiveAndDownReplicas() {
  return (liveNodes, collectionState) -> {
    for (Replica r:collectionState.getReplicas()) {
      if (r.getState() != Replica.State.DOWN && r.getState() != Replica.State.ACTIVE) {
        return false;
      }
      if (r.getState() == Replica.State.DOWN && liveNodes.contains(r.getNodeName())) {
        return false;
      }
      if (r.getState() == Replica.State.ACTIVE && !liveNodes.contains(r.getNodeName())) {
        return false;
      }
    }
    return true;
  };
}
 

private void waitTillRecovered() throws Exception {
  for (int i = 0; i < 30; i++) {
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.forceUpdateCollection("collection1");
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;
    for (Replica replica : replicas) {
      if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see recovered node");
}
 

public static List<String> getActiveReplicaCoreUrls(ZkController zkController,
                                                    String collection) {
  List<String> activeReplicaCoreUrls = new ArrayList<>();
  ClusterState clusterState = zkController.getZkStateReader().getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null && docCollection.getActiveSlices() != null && docCollection.getActiveSlices().size() > 0) {
    final Collection<Slice> activeSlices = docCollection.getActiveSlices();
    for (Slice next : activeSlices) {
      Map<String, Replica> replicasMap = next.getReplicasMap();
      if (replicasMap != null) {
        for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
          Replica replica = entry.getValue();
          if (replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
            activeReplicaCoreUrls.add(replica.getCoreUrl());
          }
        }
      }
    }
  }
  return activeReplicaCoreUrls;
}
 

protected List<Replica> getActiveOrRecoveringReplicas(String testCollectionName, String shardId) throws Exception {    
  Map<String,Replica> activeReplicas = new HashMap<String,Replica>();    
  ZkStateReader zkr = cloudClient.getZkStateReader();
  ClusterState cs = zkr.getClusterState();
  assertNotNull(cs);
  for (Slice shard : cs.getCollection(testCollectionName).getActiveSlices()) {
    if (shard.getName().equals(shardId)) {
      for (Replica replica : shard.getReplicas()) {
        final Replica.State state = replica.getState();
        if (state == Replica.State.ACTIVE || state == Replica.State.RECOVERING) {
          activeReplicas.put(replica.getName(), replica);
        }
      }
    }
  }        
  List<Replica> replicas = new ArrayList<Replica>();
  replicas.addAll(activeReplicas.values());
  return replicas;
}
 

protected static void assertAllActive(String collection, ZkStateReader zkStateReader)
    throws KeeperException, InterruptedException {

    zkStateReader.forceUpdateCollection(collection);
    ClusterState clusterState = zkStateReader.getClusterState();
    final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
    if (docCollection == null || docCollection.getSlices() == null) {
      throw new IllegalArgumentException("Cannot find collection:" + collection);
    }

    Map<String,Slice> slices = docCollection.getSlicesMap();
    for (Map.Entry<String,Slice> entry : slices.entrySet()) {
      Slice slice = entry.getValue();
      if (slice.getState() != Slice.State.ACTIVE) {
        fail("Not all shards are ACTIVE - found a shard " + slice.getName() + " that is: " + slice.getState());
      }
      Map<String,Replica> shards = slice.getReplicasMap();
      for (Map.Entry<String,Replica> shard : shards.entrySet()) {
        Replica replica = shard.getValue();
        if (replica.getState() != Replica.State.ACTIVE) {
          fail("Not all replicas are ACTIVE - found a replica " + replica.getName() + " that is: " + replica.getState());
        }
      }
    }
}
 

private void assertSliceAndReplicaCount(String collection, int numSlices, int numReplicas, int timeOutInMs) throws InterruptedException {
  TimeOut timeOut = new TimeOut(timeOutInMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
  while (!timeOut.hasTimedOut()) {
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    Collection<Slice> slices = clusterState.getCollection(collection).getActiveSlices();
    if (slices.size() == numSlices) {
      boolean isMatch = true;
      for (Slice slice : slices) {
        int count = 0;
        for (Replica replica : slice.getReplicas()) {
          if (replica.getState() == Replica.State.ACTIVE && clusterState.liveNodesContain(replica.getNodeName())) {
            count++;
          }
        }
        if (count < numReplicas) {
          isMatch = false;
        }
      }
      if (isMatch) return;
    }
    Thread.sleep(200);
  }
  fail("Expected numSlices=" + numSlices + " numReplicas=" + numReplicas + " but found " + cloudClient.getZkStateReader().getClusterState().getCollection(collection) + " with /live_nodes: " + cloudClient.getZkStateReader().getClusterState().getLiveNodes());
}
 

/**
 * Pick one replica from each shard to send the shard requests to.
 *
 * @param collection that is being queried
 * @throws IOException if an exception occurs while finding replicas
 */
protected void pickShards(String collection) throws IOException {
  try {

    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    Slice[] slices = clusterState.getCollection(collection).getActiveSlicesArr();

    for(Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName()))
        shuffler.add(replica);
      }

      Collections.shuffle(shuffler, new Random());
      Replica rep = shuffler.get(0);
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      replicaUrls.add(url);
    }
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

@ProcessElement
public void process(@Element Read spec, OutputReceiver<Read> out) throws IOException {
  ConnectionConfiguration connectionConfig = spec.getConnectionConfiguration();
  try (AuthorizedSolrClient<CloudSolrClient> client = connectionConfig.createClient()) {
    String collection = spec.getCollection();
    final ClusterState clusterState = AuthorizedSolrClient.getClusterState(client);
    DocCollection docCollection = clusterState.getCollection(collection);
    for (Slice slice : docCollection.getSlices()) {
      ArrayList<Replica> replicas = new ArrayList<>(slice.getReplicas());
      Collections.shuffle(replicas);
      // Load balancing by randomly picking an active replica
      Replica randomActiveReplica = null;
      for (Replica replica : replicas) {
        // We need to check both state of the replica and live nodes
        // to make sure that the replica is alive
        if (replica.getState() == Replica.State.ACTIVE
            && clusterState.getLiveNodes().contains(replica.getNodeName())) {
          randomActiveReplica = replica;
          break;
        }
      }
      // TODO in case of this replica goes inactive while the pipeline runs.
      // We should pick another active replica of this shard.
      checkState(
          randomActiveReplica != null,
          "Can not found an active replica for slice %s",
          slice.getName());
      out.output(spec.withReplicaInfo(ReplicaInfo.create(checkNotNull(randomActiveReplica))));
    }
  }
}
 

private String getCoreUrl(String collectionName,
                          String origCorename, ClusterState clusterState, List<Slice> slices,
                          boolean byCoreName, boolean activeReplicas) {
  String coreUrl;
  Set<String> liveNodes = clusterState.getLiveNodes();
  Collections.shuffle(slices, random);

  for (Slice slice : slices) {
    List<Replica> randomizedReplicas = new ArrayList<>(slice.getReplicas());
    Collections.shuffle(randomizedReplicas, random);

    for (Replica replica : randomizedReplicas) {
      if (!activeReplicas || (liveNodes.contains(replica.getNodeName())
          && replica.getState() == Replica.State.ACTIVE)) {

        if (byCoreName && !origCorename.equals(replica.getStr(CORE_NAME_PROP))) {
          // if it's by core name, make sure they match
          continue;
        }
        if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
          // don't count a local core
          continue;
        }

        if (origCorename != null) {
          coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
        } else {
          coreUrl = replica.getCoreUrl();
          if (coreUrl.endsWith("/")) {
            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
          }
        }

        return coreUrl;
      }
    }
  }
  return null;
}
 

protected List<String> getShardUrls() throws IOException {
  try {
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

    Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    List<String> baseUrls = new ArrayList<>();
    for(Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
          shuffler.add(replica);
        }
      }

      Collections.shuffle(shuffler, new Random());
      Replica rep = shuffler.get(0);
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      baseUrls.add(url);
    }

    return baseUrls;
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

public void publishActiveIfRegisteredAndNotActive(SolrCore core) throws Exception {
  if (core.getCoreDescriptor().getCloudDescriptor().hasRegistered()) {
    ZkStateReader zkStateReader = zkController.getZkStateReader();
    zkStateReader.forceUpdateCollection(collection);
    ClusterState clusterState = zkStateReader.getClusterState();
    Replica rep = getReplica(clusterState, collection, leaderProps.getStr(ZkStateReader.CORE_NODE_NAME_PROP));
    if (rep == null) return;
    if (rep.getState() != Replica.State.ACTIVE || core.getCoreDescriptor().getCloudDescriptor().getLastPublished() != Replica.State.ACTIVE) {
      log.debug("We have become the leader after core registration but are not in an ACTIVE state - publishing ACTIVE");
      zkController.publish(core.getCoreDescriptor(), Replica.State.ACTIVE);
    }
  }
}
 

private void waitTillNodesActive() throws Exception {
  for (int i = 0; i < 60; i++) {
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;

    Collection<String> nodesDownNames =
        nodesDown.stream()
            .map(n -> n.coreNodeName)
            .collect(Collectors.toList());

    Collection<Replica> replicasToCheck =
        replicas.stream()
            .filter(r -> !nodesDownNames.contains(r.getName()))
            .collect(Collectors.toList());

    for (Replica replica : replicasToCheck) {
      if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see all nodes active");
}
 

private Replica getSystemCollReplica() {
  ZkStateReader zkStateReader = this.coreContainer.getZkController().getZkStateReader();
  ClusterState cs = zkStateReader.getClusterState();
  DocCollection coll = cs.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);
  if (coll == null)
    throw new SolrException(SERVICE_UNAVAILABLE, CollectionAdminParams.SYSTEM_COLL + " collection not available");
  ArrayList<Slice> slices = new ArrayList<>(coll.getActiveSlices());
  if (slices.isEmpty())
    throw new SolrException(SERVICE_UNAVAILABLE, "No active slices for " + CollectionAdminParams.SYSTEM_COLL + " collection");
  Collections.shuffle(slices, RANDOM); //do load balancing

  Replica replica = null;
  for (Slice slice : slices) {
    List<Replica> replicas = new ArrayList<>(slice.getReplicasMap().values());
    Collections.shuffle(replicas, RANDOM);
    for (Replica r : replicas) {
      if (r.getState() == Replica.State.ACTIVE) {
        if (zkStateReader.getClusterState().getLiveNodes().contains(r.get(ZkStateReader.NODE_NAME_PROP))) {
          replica = r;
          break;
        } else {
          if (log.isInfoEnabled()) {
            log.info("replica {} says it is active but not a member of live nodes", r.get(ZkStateReader.NODE_NAME_PROP));
          }
        }
      }
    }
  }
  if (replica == null) {
    throw new SolrException(SERVICE_UNAVAILABLE, "No active replica available for " + CollectionAdminParams.SYSTEM_COLL + " collection");
  }
  return replica;
}
 

private void getPersistedCheckpoints() throws IOException {
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
  Slice[] slices = CloudSolrStream.getSlices(checkpointCollection, zkStateReader, false);

  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();

  OUTER:
  for(Slice slice : slices) {
    Collection<Replica> replicas = slice.getReplicas();
    for(Replica replica : replicas) {
      if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())){
        HttpSolrClient httpClient = streamContext.getSolrClientCache().getHttpSolrClient(replica.getCoreUrl());
        try {
          SolrDocument doc = httpClient.getById(id);
          if(doc != null) {
            @SuppressWarnings({"unchecked"})
            List<String> checkpoints = (List<String>)doc.getFieldValue("checkpoint_ss");
            for (String checkpoint : checkpoints) {
              String[] pair = checkpoint.split("~");
              this.checkpoints.put(pair[0], Long.parseLong(pair[1]));
            }
          }
        } catch (Exception e) {
          throw new IOException(e);
        }
        break OUTER;
      }
    }
  }
}
 

private long getCheckpoint(Slice slice, Set<String> liveNodes) throws IOException {
  Collection<Replica> replicas = slice.getReplicas();
  long checkpoint = -1;
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set("q","*:*");
  params.set(SORT, "_version_ desc");
  params.set(DISTRIB, "false");
  params.set("rows", 1);
  for(Replica replica : replicas) {
    if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
      String coreUrl = replica.getCoreUrl();
      SolrStream solrStream = new SolrStream(coreUrl, params);

      if(streamContext != null) {
        StreamContext localContext = new StreamContext();
        localContext.setSolrClientCache(streamContext.getSolrClientCache());
        localContext.setObjectCache(streamContext.getObjectCache());
        solrStream.setStreamContext(localContext);
      }

      try {
        solrStream.open();
        Tuple tuple = solrStream.read();
        if(tuple.EOF) {
          return 0;
        } else {
          checkpoint = tuple.getLong("_version_");
        }
        break;
      } finally {
        solrStream.close();
      }
    }
  }
  return checkpoint;
}
 

private boolean isActive(Replica replica) {
  return replica.getState() == Replica.State.ACTIVE;
}
 

/**
 * Take action when we failed to create a SolrCore. If error is due to corrupt index, try to recover. Various recovery
 * strategies can be specified via system properties "-DCoreInitFailedAction={fromleader, none}"
 *
 * @param original   the problem seen when loading the core the first time.
 * @param dcore      core descriptor for the core to create
 * @param coreConfig core config for the core to create
 * @return if possible
 * @throws SolrException rethrows the original exception if we will not attempt to recover, throws a new SolrException with the
 *                       original exception as a suppressed exception if there is a second problem creating the solr core.
 * @see CoreInitFailedAction
 */
private SolrCore processCoreCreateException(SolrException original, CoreDescriptor dcore, ConfigSet coreConfig) {
  // Traverse full chain since CIE may not be root exception
  Throwable cause = original;
  while ((cause = cause.getCause()) != null) {
    if (cause instanceof CorruptIndexException) {
      break;
    }
  }

  // If no CorruptIndexException, nothing we can try here
  if (cause == null) throw original;

  CoreInitFailedAction action = CoreInitFailedAction.valueOf(System.getProperty(CoreInitFailedAction.class.getSimpleName(), "none"));
  log.debug("CorruptIndexException while creating core, will attempt to repair via {}", action);

  switch (action) {
    case fromleader: // Recovery from leader on a CorruptedIndexException
      if (isZooKeeperAware()) {
        CloudDescriptor desc = dcore.getCloudDescriptor();
        try {
          Replica leader = getZkController().getClusterState()
              .getCollection(desc.getCollectionName())
              .getSlice(desc.getShardId())
              .getLeader();
          if (leader != null && leader.getState() == State.ACTIVE) {
            log.info("Found active leader, will attempt to create fresh core and recover.");
            resetIndexDirectory(dcore, coreConfig);
            // the index of this core is emptied, its term should be set to 0
            getZkController().getShardTerms(desc.getCollectionName(), desc.getShardId()).setTermToZero(desc.getCoreNodeName());
            return new SolrCore(this, dcore, coreConfig);
          }
        } catch (SolrException se) {
          se.addSuppressed(original);
          throw se;
        }
      }
      throw original;
    case none:
      throw original;
    default:
      log.warn("Failed to create core, and did not recognize specified 'CoreInitFailedAction': [{}]. Valid options are {}.",
          action, Arrays.asList(CoreInitFailedAction.values()));
      throw original;
  }
}
 

protected void constructStreams() throws IOException {
  try {
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
    Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

    ModifiableSolrParams mParams = new ModifiableSolrParams(params);
    mParams.set(DISTRIB, "false"); // We are the aggregator.
    String fl = mParams.get("fl");
    mParams.set(SORT, "_version_ asc");
    if(!fl.contains(VERSION_FIELD)) {
      fl += ",_version_";
    }
    mParams.set("fl", fl);

    Random random = new Random();

    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    for(Slice slice : slices) {
      ModifiableSolrParams localParams = new ModifiableSolrParams(mParams);
      long checkpoint = checkpoints.get(slice.getName());

      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName()))
          shuffler.add(replica);
      }

      Replica rep = shuffler.get(random.nextInt(shuffler.size()));
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      SolrStream solrStream = new SolrStream(url, localParams);
      solrStream.setSlice(slice.getName());
      solrStream.setCheckpoint(checkpoint);
      solrStream.setTrace(true);
      if(streamContext != null) {
        solrStream.setStreamContext(streamContext);
      }
      solrStreams.add(solrStream);
    }
  } catch (Exception e) {
    throw new IOException(e);
  }
}