Java Code Examples for org.apache.solr.common.cloud.DocCollection#getSlice()

@Test
public void testBuildClusterState_Simple() {
  try (ZkStateReader zkStateReader = ClusterStateMockUtil.buildClusterState("csr", "baseUrl1_")) {
    ClusterState clusterState = zkStateReader.getClusterState();
    assertNotNull(clusterState);
    assertEquals(1, clusterState.getCollectionStates().size());
    DocCollection collection1 = clusterState.getCollectionOrNull("collection1");
    assertNotNull(collection1);
    assertEquals(DocRouter.DEFAULT, collection1.getRouter());
    assertEquals(1, collection1.getActiveSlices().size());
    assertEquals(1, collection1.getSlices().size());
    Slice slice1 = collection1.getSlice("slice1");
    assertNotNull(slice1);
    assertEquals(1, slice1.getReplicas().size());
    Replica replica1 = slice1.getReplica("replica1");
    assertNotNull(replica1);
    assertEquals("baseUrl1_", replica1.getNodeName());
    assertEquals("slice1_replica1", replica1.getCoreName());
    assertEquals("http://baseUrl1", replica1.getBaseUrl());
    assertEquals("http://baseUrl1/slice1_replica1/", replica1.getCoreUrl());
    assertEquals(Replica.State.ACTIVE, replica1.getState());
    assertEquals(Replica.Type.NRT, replica1.getType());
  }
}
 

/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
protected boolean amISubShardLeader(DocCollection coll, Slice parentSlice, String id, SolrInputDocument doc) throws InterruptedException {
  // Am I the leader of a shard in "construction/recovery" state?
  String myShardId = cloudDesc.getShardId();
  Slice mySlice = coll.getSlice(myShardId);
  final Slice.State state = mySlice.getState();
  if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) {
    Replica myLeader = zkController.getZkStateReader().getLeaderRetry(collection, myShardId);
    boolean amILeader = myLeader.getName().equals(cloudDesc.getCoreNodeName());
    if (amILeader) {
      // Does the document belong to my hash range as well?
      DocRouter.Range myRange = mySlice.getRange();
      if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      if (parentSlice != null)  {
        boolean isSubset = parentSlice.getRange() != null && myRange.isSubsetOf(parentSlice.getRange());
        return isSubset && coll.getRouter().isTargetSlice(id, doc, req.getParams(), myShardId, coll);
      } else  {
        // delete by query case -- as long as I am a sub shard leader we're fine
        return true;
      }
    }
  }
  return false;
}
 

private void verifyShard(DocCollection coll, String shard, Slice.State expectedState, int numNrt, int numTlog, int numPull) throws Exception {
  Slice s = coll.getSlice(shard);
  assertEquals("unexpected shard state", expectedState, s.getState());
  AtomicInteger actualNrt = new AtomicInteger();
  AtomicInteger actualTlog = new AtomicInteger();
  AtomicInteger actualPull = new AtomicInteger();
  s.getReplicas().forEach(r -> {
    switch (r.getType()) {
      case NRT: actualNrt.incrementAndGet(); break;
      case TLOG: actualTlog.incrementAndGet(); break;
      case PULL: actualPull.incrementAndGet(); break;
    }
  });
  assertEquals("actual NRT", numNrt, actualNrt.get());
  assertEquals("actual TLOG", numTlog, actualTlog.get());
  assertEquals("actual PULL", numPull, actualPull.get());
}
 

/**
 * Set UpdateLog to buffer updates if the slice is in construction.
 */
private void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) {

  if (coreContainer != null && coreContainer.isZooKeeperAware()) {
    if (reqHandlers.get("/get") == null) {
      log.warn("WARNING: RealTimeGetHandler is not registered at /get. SolrCloud will always use full index replication instead of the more efficient PeerSync method.");
    }

    // ZK pre-register would have already happened so we read slice properties now
    final ClusterState clusterState = coreContainer.getZkController().getClusterState();
    final DocCollection collection = clusterState.getCollection(coreDescriptor.getCloudDescriptor().getCollectionName());
    final Slice slice = collection.getSlice(coreDescriptor.getCloudDescriptor().getShardId());
    if (slice.getState() == Slice.State.CONSTRUCTION) {
      // set update log to buffer before publishing the core
      getUpdateHandler().getUpdateLog().bufferUpdates();
    }
  }
}
 

static List<String> verifyReplicaAvailability(List<ZkNodeProps> sourceReplicas, ClusterState state) {
  List<String> res = new ArrayList<>();
  for (ZkNodeProps sourceReplica : sourceReplicas) {
    String coll = sourceReplica.getStr(COLLECTION_PROP);
    String shard = sourceReplica.getStr(SHARD_ID_PROP);
    String replicaName = sourceReplica.getStr(ZkStateReader.REPLICA_PROP);
    DocCollection collection = state.getCollection(coll);
    Slice slice = collection.getSlice(shard);
    if (slice.getReplicas().size() < 2) {
      // can't delete the only replica in existence
      res.add(coll + "/" + shard + "/" + replicaName + ", type=" + sourceReplica.getStr(ZkStateReader.REPLICA_TYPE));
    } else { // check replica types
      int otherNonPullReplicas = 0;
      for (Replica r : slice.getReplicas()) {
        if (!r.getName().equals(replicaName) && !r.getType().equals(Replica.Type.PULL)) {
          otherNonPullReplicas++;
        }
      }
      // can't delete - there are no other non-pull replicas
      if (otherNonPullReplicas == 0) {
        res.add(coll + "/" + shard + "/" + replicaName + ", type=" + sourceReplica.getStr(ZkStateReader.REPLICA_TYPE));
      }
    }
  }
  return res;
}
 

@Test
public void testBuildClusterState_ReplicaStateAndType() {
  try (ZkStateReader zkStateReader = ClusterStateMockUtil.buildClusterState("csrStRpDnF", "baseUrl1_")) {
    ClusterState clusterState = zkStateReader.getClusterState();
    assertNotNull(clusterState);
    assertEquals(1, clusterState.getCollectionStates().size());
    DocCollection collection1 = clusterState.getCollectionOrNull("collection1");
    assertNotNull(collection1);
    assertEquals(DocRouter.DEFAULT, collection1.getRouter());
    assertEquals(1, collection1.getActiveSlices().size());
    assertEquals(1, collection1.getSlices().size());
    Slice slice1 = collection1.getSlice("slice1");
    assertNotNull(slice1);
    assertEquals(4, slice1.getReplicas().size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.NRT && replica.getState() == Replica.State.ACTIVE).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.NRT && replica.getState() == Replica.State.RECOVERY_FAILED).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.TLOG && replica.getState() == Replica.State.RECOVERING).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.PULL && replica.getState() == Replica.State.DOWN).size());
  }
}
 

@Test
public void testBuildClusterState_ReplicaTypes() {
  try (ZkStateReader zkStateReader = ClusterStateMockUtil.buildClusterState("csntp", "baseUrl1_")) {
    ClusterState clusterState = zkStateReader.getClusterState();
    assertNotNull(clusterState);
    assertEquals(1, clusterState.getCollectionStates().size());
    DocCollection collection1 = clusterState.getCollectionOrNull("collection1");
    assertNotNull(collection1);
    assertEquals(DocRouter.DEFAULT, collection1.getRouter());
    assertEquals(1, collection1.getActiveSlices().size());
    assertEquals(1, collection1.getSlices().size());
    Slice slice1 = collection1.getSlice("slice1");
    assertNotNull(slice1);
    assertEquals(3, slice1.getReplicas().size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.NRT).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.TLOG).size());
    assertEquals(1, slice1.getReplicas(replica -> replica.getType() == Replica.Type.PULL).size());
  }
}
 

public ZkWriteCommand createShard(final ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  String shardId = message.getStr(ZkStateReader.SHARD_ID_PROP);
  DocCollection collection = clusterState.getCollection(collectionName);
  Slice slice = collection.getSlice(shardId);
  if (slice == null) {
    @SuppressWarnings({"unchecked"})
    Map<String, Replica> replicas = Collections.EMPTY_MAP;
    Map<String, Object> sliceProps = new HashMap<>();
    String shardRange = message.getStr(ZkStateReader.SHARD_RANGE_PROP);
    String shardState = message.getStr(ZkStateReader.SHARD_STATE_PROP);
    String shardParent = message.getStr(ZkStateReader.SHARD_PARENT_PROP);
    String shardParentZkSession = message.getStr("shard_parent_zk_session");
    String shardParentNode = message.getStr("shard_parent_node");
    sliceProps.put(Slice.RANGE, shardRange);
    sliceProps.put(ZkStateReader.STATE_PROP, shardState);
    if (shardParent != null) {
      sliceProps.put(Slice.PARENT, shardParent);
    }
    if (shardParentZkSession != null) {
      sliceProps.put("shard_parent_zk_session", shardParentZkSession);
    }
    if (shardParentNode != null)  {
      sliceProps.put("shard_parent_node", shardParentNode);
    }
    collection = updateSlice(collectionName, collection, new Slice(shardId, replicas, sliceProps, collectionName));
    return new ZkWriteCommand(collectionName, collection);
  } else {
    log.error("Unable to create Shard: {} because it already exists in collection: {}", shardId, collectionName);
    return ZkStateWriter.NO_OP;
  }
}
 

public ZkWriteCommand removeRoutingRule(final ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
  String routeKeyStr = message.getStr("routeKey");

  log.info("Overseer.removeRoutingRule invoked for collection: {} shard: {} routeKey: {}"
      , collectionName, shard, routeKeyStr);

  DocCollection collection = clusterState.getCollection(collectionName);
  Slice slice = collection.getSlice(shard);
  if (slice == null) {
    log.warn("Unknown collection: {} shard: {}", collectionName, shard);
    return ZkStateWriter.NO_OP;
  }
  Map<String, RoutingRule> routingRules = slice.getRoutingRules();
  if (routingRules != null) {
    routingRules.remove(routeKeyStr); // no rules left
    Map<String, Object> props = slice.shallowCopy();
    props.put("routingRules", routingRules);
    Slice newSlice = new Slice(slice.getName(), slice.getReplicasCopy(), props,collectionName);
    return new ZkWriteCommand(collectionName,
        CollectionMutator.updateSlice(collectionName, collection, newSlice));
  }

  return ZkStateWriter.NO_OP;
}
 

public ZkWriteCommand updateShardState(ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  log.info("Update shard state invoked for collection: {} with message: {}", collectionName, message);

  DocCollection collection = clusterState.getCollection(collectionName);
  Map<String, Slice> slicesCopy = new LinkedHashMap<>(collection.getSlicesMap());
  for (String key : message.keySet()) {
    if (ZkStateReader.COLLECTION_PROP.equals(key)) continue;
    if (Overseer.QUEUE_OPERATION.equals(key)) continue;

    Slice slice = collection.getSlice(key);
    if (slice == null) {
      throw new RuntimeException("Overseer.updateShardState unknown collection: " + collectionName + " slice: " + key);
    }
    if (log.isInfoEnabled()) {
      log.info("Update shard state {} to {}", key, message.getStr(key));
    }
    Map<String, Object> props = slice.shallowCopy();
    
    if (Slice.State.getState(message.getStr(key)) == Slice.State.ACTIVE) {
      props.remove(Slice.PARENT);
      props.remove("shard_parent_node");
      props.remove("shard_parent_zk_session");
    }
    props.put(ZkStateReader.STATE_PROP, message.getStr(key));
    // we need to use epoch time so that it's comparable across Overseer restarts
    props.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs()));
    Slice newSlice = new Slice(slice.getName(), slice.getReplicasCopy(), props,collectionName);
    slicesCopy.put(slice.getName(), newSlice);
  }

  return new ZkWriteCommand(collectionName, collection.copyWithSlices(slicesCopy));
}
 

public static boolean replicaExists(ClusterState clusterState, String collection, String shard, String coreNodeName) {
  DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null) {
    Slice slice = docCollection.getSlice(shard);
    if (slice != null) {
      return slice.getReplica(coreNodeName) != null;
    }
  }
  return false;
}
 

public static Slice getParentSlice(ClusterState clusterState, String collectionName, AtomicReference<String> slice, String splitKey) {
  DocCollection collection = clusterState.getCollection(collectionName);
  DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;

  Slice parentSlice;

  if (slice.get() == null) {
    if (router instanceof CompositeIdRouter) {
      Collection<Slice> searchSlices = router.getSearchSlicesSingle(splitKey, new ModifiableSolrParams(), collection);
      if (searchSlices.isEmpty()) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to find an active shard for split.key: " + splitKey);
      }
      if (searchSlices.size() > 1) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
            "Splitting a split.key: " + splitKey + " which spans multiple shards is not supported");
      }
      parentSlice = searchSlices.iterator().next();
      slice.set(parentSlice.getName());
      log.info("Split by route.key: {}, parent shard is: {} ", splitKey, slice);
    } else {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "Split by route key can only be used with CompositeIdRouter or subclass. Found router: "
              + router.getClass().getName());
    }
  } else {
    parentSlice = collection.getSlice(slice.get());
  }

  if (parentSlice == null) {
    // no chance of the collection being null because ClusterState#getCollection(String) would have thrown
    // an exception already
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shard with the specified name exists: " + slice);
  }
  return parentSlice;
}
 

public static String buildSolrCoreName(DistribStateManager stateManager, DocCollection collection, String shard, Replica.Type type, boolean newCollection) {
  Slice slice = collection.getSlice(shard);
  int defaultValue = defaultCounterValue(collection, newCollection, shard);
  int replicaNum = incAndGetId(stateManager, collection.getName(), defaultValue);
  String coreName = buildSolrCoreName(collection.getName(), shard, type, replicaNum);
  while (existCoreName(coreName, slice)) {
    replicaNum = incAndGetId(stateManager, collection.getName(), defaultValue);
    coreName = buildSolrCoreName(collection.getName(), shard, type, replicaNum);
  }
  return coreName;
}
 

@SuppressWarnings("unchecked")
void deleteReplica(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results, Runnable onComplete)
        throws KeeperException, InterruptedException {
  if (log.isDebugEnabled()) {
    log.debug("deleteReplica() : {}", Utils.toJSONString(message));
  }
  boolean parallel = message.getBool("parallel", false);

  //If a count is specified the strategy needs be different
  if (message.getStr(COUNT_PROP) != null) {
    deleteReplicaBasedOnCount(clusterState, message, results, onComplete, parallel);
    return;
  }


  ocmh.checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP);
  String extCollectionName = message.getStr(COLLECTION_PROP);
  String shard = message.getStr(SHARD_ID_PROP);
  String replicaName = message.getStr(REPLICA_PROP);

  boolean followAliases = message.getBool(FOLLOW_ALIASES, false);
  String collectionName;
  if (followAliases) {
    collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName);
  } else {
    collectionName = extCollectionName;
  }

  DocCollection coll = clusterState.getCollection(collectionName);
  Slice slice = coll.getSlice(shard);
  if (slice == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
            "Invalid shard name : " +  shard + " in collection : " +  collectionName);
  }

  deleteCore(slice, collectionName, replicaName, message, shard, results, onComplete,  parallel);

}
 

private Replica getReplicaOrNull(DocCollection docCollection, String shard, String coreNodeName) {
  if (docCollection == null) return null;

  Slice slice = docCollection.getSlice(shard);
  if (slice == null) return null;

  Replica replica = slice.getReplica(coreNodeName);
  if (replica == null) return null;
  if (!getNodeName().equals(replica.getNodeName())) return null;

  return replica;
}
 

/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
private boolean couldIbeSubShardLeader(DocCollection coll) {
  // Could I be the leader of a shard in "construction/recovery" state?
  String myShardId = cloudDesc.getShardId();
  Slice mySlice = coll.getSlice(myShardId);
  Slice.State state = mySlice.getState();
  return state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY;
}
 

private boolean canKillIndexer(String sliceName) throws KeeperException, InterruptedException {
  int numIndexersFoundInShard = 0;
  for (CloudJettyRunner cloudJetty : shardToJetty.get(sliceName)) {
    
    // get latest cloud state
    zkStateReader.forceUpdateCollection(collection);
    
    DocCollection docCollection = zkStateReader.getClusterState().getCollection(collection);
    
    Slice slice = docCollection.getSlice(sliceName);
    
    ZkNodeProps props = slice.getReplicasMap().get(cloudJetty.coreNodeName);
    if (props == null) {
      throw new RuntimeException("shard name " + cloudJetty.coreNodeName + " not found in " + slice.getReplicasMap().keySet());
    }
    
    final Replica.State state = Replica.State.getState(props.getStr(ZkStateReader.STATE_PROP));
    final Replica.Type replicaType = Replica.Type.valueOf(props.getStr(ZkStateReader.REPLICA_TYPE));
    final String nodeName = props.getStr(ZkStateReader.NODE_NAME_PROP);
    
    if (cloudJetty.jetty.isRunning()
        && state == Replica.State.ACTIVE
        && (replicaType == Replica.Type.TLOG || replicaType == Replica.Type.NRT) 
        && zkStateReader.getClusterState().liveNodesContain(nodeName)) {
      numIndexersFoundInShard++;
    }
  }
  return numIndexersFoundInShard > 1;
}
 

private String getBaseUrl() {
  DocCollection collection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(collectionName);
  Slice slice = collection.getSlice("shard1");
  return slice.getLeader().getCoreUrl();
}
 

private void doTestSetArbitraryPropertySliceUnique(String propIn) throws InterruptedException, KeeperException, IOException, SolrServerException {
  final String prop = (random().nextBoolean()) ? propIn : propIn.toUpperCase(Locale.ROOT);
  // First set the property in some replica in some slice
  forceUpdateCollectionStatus();
  DocCollection docCollection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);

  Slice[] slices = docCollection.getSlices().toArray(new Slice[0]);
  Slice slice = slices[random().nextInt(slices.length)];

  // Bounce around a bit setting this property and insure it's only set in one replica.
  Replica[] reps = slice.getReplicas().toArray(new Replica[0]);
  for (int idx = 0; idx < 4; ++idx) {
    Replica rep = reps[random().nextInt(reps.length)];
    // Set the property on a particular replica
    setProp(slice, rep, prop);
    TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);

    long count = 0;
    boolean rightRep = false;
    Slice modSlice;
    DocCollection modColl = null; // keeps IDE happy

    // insure that no other replica in that slice has the property when we return.
    while (timeout.hasTimedOut() == false) {
      forceUpdateCollectionStatus();
      modColl = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);
      modSlice = modColl.getSlice(slice.getName());
      rightRep = modSlice.getReplica(rep.getName()).getBool("property." + prop.toLowerCase(Locale.ROOT), false);
      count = modSlice.getReplicas().stream().filter(thisRep -> thisRep.getBool("property." + prop.toLowerCase(Locale.ROOT), false)).count();

      if (count == 1 && rightRep) {
        break;
      }

      TimeUnit.MILLISECONDS.sleep(100);
    }
    if (count != 1 || rightRep == false) {
      fail("The property " + prop + " was not uniquely distributed in slice " + slice.getName()
          + " " + modColl.toString());
    }
  }
}
 

private boolean waitForReplicasToComeUp(int timeoutms) throws InterruptedException {
  long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutms, TimeUnit.MILLISECONDS);
  final String shardsElectZkPath = electionPath + LeaderElector.ELECTION_NODE;

  DocCollection docCollection = zkController.getClusterState().getCollectionOrNull(collection);
  Slice slices = (docCollection == null) ? null : docCollection.getSlice(shardId);
  int cnt = 0;
  while (!isClosed && !cc.isShutDown()) {
    // wait for everyone to be up
    if (slices != null) {
      int found = 0;
      try {
        found = zkClient.getChildren(shardsElectZkPath, null, true).size();
      } catch (KeeperException e) {
        if (e instanceof KeeperException.SessionExpiredException) {
          // if the session has expired, then another election will be launched, so
          // quit here
          throw new SolrException(ErrorCode.SERVER_ERROR,
              "ZK session expired - cancelling election for " + collection + " " + shardId);
        }
        SolrException.log(log,
            "Error checking for the number of election participants", e);
      }

      // on startup and after connection timeout, wait for all known shards
      if (found >= slices.getReplicas(EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT)).size()) {
        log.info("Enough replicas found to continue.");
        return true;
      } else {
        if (cnt % 40 == 0) {
          if (log.isInfoEnabled()) {
            log.info("Waiting until we see more replicas up for shard {}: total={} found={} timeoute in={}ms"
                , shardId, slices.getReplicas(EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT)).size(), found,
                TimeUnit.MILLISECONDS.convert(timeoutAt - System.nanoTime(), TimeUnit.NANOSECONDS));
          }
        }
      }

      if (System.nanoTime() > timeoutAt) {
        log.info("Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later");
        return false;
      }
    } else {
      log.warn("Shard not found: {} for collection {}", shardId, collection);

      return false;

    }

    Thread.sleep(500);
    docCollection = zkController.getClusterState().getCollectionOrNull(collection);
    slices = (docCollection == null) ? null : docCollection.getSlice(shardId);
    cnt++;
  }
  return false;
}