Java Code Examples for org.apache.solr.common.cloud.Replica#getNodeName()

@Test
@ShardsFixed(num = 4)
public void test() throws Exception {
  waitForRecoveriesToFinish(false);
  assertEquals(4, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlices().size());
  index("id", "a!doc1");  // shard3
  index("id", "b!doc1");  // shard1
  index("id", "c!doc1");  // shard2
  index("id", "e!doc1");  // shard4
  commit();

  doQuery("a!doc1", "q", "*:*", ShardParams._ROUTE_, "a!"); // can go to any random node

  // query shard3 directly with _route_=a! so that we trigger the short circuited request path
  Replica shard3 = cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getLeader("shard3");
  String nodeName = shard3.getNodeName();
  SolrClient shard3Client = getClient(nodeName);
  QueryResponse response = shard3Client.query(new SolrQuery("*:*").add(ShardParams._ROUTE_, "a!").add(ShardParams.SHARDS_INFO, "true"));

  assertEquals("Could not find doc", 1, response.getResults().getNumFound());
  NamedList<?> sinfo = (NamedList<?>) response.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("missing shard info for short circuited request", sinfo);
}
 

public CloudJettyRunner(JettySolrRunner jetty, Replica replica,
                        String collection, String shard, String coreNodeName) {
  this.jetty = jetty;
  this.info = replica;
  this.collection = collection;
  
  Properties nodeProperties = jetty.getNodeProperties();

  // we need to update the jetty's shard so that it registers itself to the right shard when restarted
  this.shard = shard;
  nodeProperties.setProperty(CoreDescriptor.CORE_SHARD, this.shard);

  // we need to update the jetty's shard so that it registers itself under the right core name when restarted
  this.coreNodeName = coreNodeName;
  nodeProperties.setProperty(CoreDescriptor.CORE_NODE_NAME, this.coreNodeName);

  this.nodeName = replica.getNodeName();

  ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(info);
  this.url = coreNodeProps.getCoreUrl();

  // strip the trailing slash as this can cause issues when executing requests
  this.client = createNewSolrServer(this.url.substring(0, this.url.length() - 1));
}
 

public ReplicaInfo(String coll, String shard, Replica r, Map<String, Object> vals) {
  this.name = r.getName();
  this.core = r.getCoreName();
  this.collection = coll;
  this.shard = shard;
  this.type = r.getType();
  this.node = r.getNodeName();
  boolean maybeLeader = r.getBool(LEADER_PROP, false);
  if (vals != null) {
    this.variables.putAll(vals);
    maybeLeader = "true".equals(String.valueOf(vals.getOrDefault(LEADER_PROP, maybeLeader)));
  }
  this.isLeader = maybeLeader;
  validate();
}
 

protected int getReplicaPort(Replica replica) {
  String replicaNode = replica.getNodeName();
  String tmp = replicaNode.substring(replicaNode.indexOf(':')+1);
  if (tmp.indexOf('_') != -1)
    tmp = tmp.substring(0,tmp.indexOf('_'));
  return Integer.parseInt(tmp);
}
 

private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
    ClusterState clusterState = getCommonCloudSolrClient().getZkStateReader().getClusterState();
    
    int expectedSlices = numShardsNumReplicaList.get(0);
    // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
    int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
    int expectedTotalShards = expectedSlices * expectedShardsPerSlice;

//      Map<String,DocCollection> collections = clusterState
//          .getCollectionStates();
      if (clusterState.hasCollection(collectionName)) {
        Map<String,Slice> slices = clusterState.getCollection(collectionName).getSlicesMap();
        // did we find expectedSlices slices/shards?
      if (slices.size() != expectedSlices) {
        return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
      }
      int totalShards = 0;
      for (String sliceName : slices.keySet()) {
        for (Replica replica : slices.get(sliceName).getReplicas()) {
          if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
            return "Shard " + replica.getName() + " created on node " + replica.getNodeName() + " not allowed to run shards for the created collection " + collectionName;
          }
        }
        totalShards += slices.get(sliceName).getReplicas().size();
      }
      if (totalShards != expectedTotalShards) {
        return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
        }
      return null;
    } else {
      return "Could not find new collection " + collectionName;
    }
  }
 

private List<ZkNodeProps> getReplicasForSlice(String collectionName, Slice slice) {
  List<ZkNodeProps> sourceReplicas = new ArrayList<>();
  for (Replica replica : slice.getReplicas()) {
    ZkNodeProps props = new ZkNodeProps(
        COLLECTION_PROP, collectionName,
        SHARD_ID_PROP, slice.getName(),
        ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
        ZkStateReader.REPLICA_PROP, replica.getName(),
        CoreAdminParams.NODE, replica.getNodeName());
    sourceReplicas.add(props);
  }
  return sourceReplicas;
}
 

public static void checkDiskSpace(String collection, String shard, Replica parentShardLeader, SolrIndexSplitter.SplitMethod method, SolrCloudManager cloudManager) throws SolrException {
  // check that enough disk space is available on the parent leader node
  // otherwise the actual index splitting will always fail
  NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
  Map<String, Object> nodeValues = nodeStateProvider.getNodeValues(parentShardLeader.getNodeName(),
      Collections.singletonList(ImplicitSnitch.DISK));
  Map<String, Map<String, List<ReplicaInfo>>> infos = nodeStateProvider.getReplicaInfo(parentShardLeader.getNodeName(),
      Collections.singletonList(Type.CORE_IDX.metricsAttribute));
  if (infos.get(collection) == null || infos.get(collection).get(shard) == null) {
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing replica information for parent shard leader");
  }
  // find the leader
  List<ReplicaInfo> lst = infos.get(collection).get(shard);
  Double indexSize = null;
  for (ReplicaInfo info : lst) {
    if (info.getCore().equals(parentShardLeader.getCoreName())) {
      Number size = (Number)info.getVariable(Type.CORE_IDX.metricsAttribute);
      if (size == null) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing index size information for parent shard leader");
      }
      indexSize = (Double) Type.CORE_IDX.convertVal(size);
      break;
    }
  }
  if (indexSize == null) {
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing replica information for parent shard leader");
  }
  Number freeSize = (Number)nodeValues.get(ImplicitSnitch.DISK);
  if (freeSize == null) {
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing node disk space information for parent shard leader");
  }
  // 100% more for REWRITE, 5% more for LINK
  double neededSpace = method == SolrIndexSplitter.SplitMethod.REWRITE ? 2.0 * indexSize : 1.05 * indexSize;
  if (freeSize.doubleValue() < neededSpace) {
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "not enough free disk space to perform index split on node " +
        parentShardLeader.getNodeName() + ", required: " + neededSpace + ", available: " + freeSize);
  }
}
 

public void testCreateCollection() throws Exception {
  String prefix = "testCreateCollection";
  String xyz = prefix + "_xyz";
  String abc = prefix + "_abc";

  CloudSolrClient solrClient = cluster.getSolrClient();

  String setClusterPolicyCommand = "{" +
      " 'set-cluster-policy': [" +
      "      {'cores':'<10', 'node':'#ANY'}," +
      "    ]" +
      "}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
  solrClient.request(req);

  String chosenNode = cluster.getRandomJetty(random()).getNodeName();
  CollectionAdminRequest.createCollection(abc, 1, 1)
      .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
      .process(solrClient);
  CollectionAdminRequest.createCollection(xyz, 1, 1)
      .setWithCollection(abc)
      .process(solrClient);

  DocCollection c1 = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(xyz);
  assertNotNull(c1);
  assertEquals(abc, c1.getStr(WITH_COLLECTION));
  Replica replica = c1.getReplicas().get(0);
  String nodeName = replica.getNodeName();

  assertEquals(chosenNode, nodeName);
}
 

protected int getReplicaPort(Replica replica) {
  String replicaNode = replica.getNodeName();    
  String tmp = replicaNode.substring(replicaNode.indexOf(':')+1);
  if (tmp.indexOf('_') != -1)
    tmp = tmp.substring(0,tmp.indexOf('_'));
  return Integer.parseInt(tmp);    
}
 

public static String getLeaderNode(MiniSolrCloudCluster cluster, String collection) throws Exception {
  for (Replica replica : cluster.getSolrClient().getClusterStateProvider().getCollection(collection).getReplicas()) {
    if (cluster.getSolrClient().getClusterStateProvider().getCollection(collection).getLeader("shard1") == replica) {
      return replica.getNodeName();
    }
  }
  return "";
}
 

private boolean collectCurrentPropStats() {
  int maxAssigned = 0;
  // Get a list of potential replicas that can host the property _and_ their counts
  // Move any obvious entries to a list of replicas to change the property on
  Set<String> allHosts = new HashSet<>();
  for (Slice slice : collection.getSlices()) {
    boolean sliceHasProp = false;
    for (Replica replica : slice.getReplicas()) {
      if (onlyActiveNodes && isActive(replica) == false) {
        if (StringUtils.isNotBlank(replica.getStr(property))) {
          removeProp(slice, replica.getName()); // Note, we won't be committing this to ZK until later.
        }
        continue;
      }
      allHosts.add(replica.getNodeName());
      String nodeName = replica.getNodeName();
      if (StringUtils.isNotBlank(replica.getStr(property))) {
        if (sliceHasProp) {
          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
              "'" + BALANCESHARDUNIQUE + "' should only be called for properties that have at most one member " +
                  "in any slice with the property set. No action taken.");
        }
        if (nodesHostingProp.containsKey(nodeName) == false) {
          nodesHostingProp.put(nodeName, new ArrayList<>());
        }
        nodesHostingProp.get(nodeName).add(new SliceReplica(slice, replica));
        ++assigned;
        maxAssigned = Math.max(maxAssigned, nodesHostingProp.get(nodeName).size());
        sliceHasProp = true;
      }
      if (nodesHostingReplicas.containsKey(nodeName) == false) {
        nodesHostingReplicas.put(nodeName, new ArrayList<>());
      }
      nodesHostingReplicas.get(nodeName).add(new SliceReplica(slice, replica));
    }
  }

  // If the total number of already-hosted properties assigned to nodes
  // that have potential to host leaders is equal to the slice count _AND_ none of the current nodes has more than
  // the max number of properties, there's nothing to do.
  origMaxPropPerNode = collection.getSlices().size() / allHosts.size();

  // Some nodes can have one more of the proeprty if the numbers aren't exactly even.
  origModulo = collection.getSlices().size() % allHosts.size();
  if (origModulo > 0) {
    origMaxPropPerNode++;  // have to have some nodes with 1 more property.
  }

  // We can say for sure that we need to rebalance if we don't have as many assigned properties as slices.
  if (assigned != collection.getSlices().size()) {
    return true;
  }

  // Make sure there are no more slices at the limit than the "leftovers"
  // Let's say there's 7 slices and 3 nodes. We need to distribute the property as 3 on node1, 2 on node2 and 2 on node3
  // (3, 2, 2) We need to be careful to not distribute them as 3, 3, 1. that's what this check is all about.
  int counter = origModulo;
  for (List<SliceReplica> list : nodesHostingProp.values()) {
    if (list.size() == origMaxPropPerNode) --counter;
  }
  if (counter == 0) return false; // nodes with 1 extra leader are exactly the needed number

  return true;
}
 

public List<ZkWriteCommand> downNode(ClusterState clusterState, ZkNodeProps message) {
  List<ZkWriteCommand> zkWriteCommands = new ArrayList<>();
  String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP);

  log.debug("DownNode state invoked for node: {}", nodeName);

  Map<String, DocCollection> collections = clusterState.getCollectionsMap();
  for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
    String collection = entry.getKey();
    DocCollection docCollection = entry.getValue();

    Map<String,Slice> slicesCopy = new LinkedHashMap<>(docCollection.getSlicesMap());

    boolean needToUpdateCollection = false;
    for (Entry<String, Slice> sliceEntry : slicesCopy.entrySet()) {
      Slice slice = sliceEntry.getValue();
      Map<String, Replica> newReplicas = slice.getReplicasCopy();

      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        String rNodeName = replica.getNodeName();
        if (rNodeName == null) {
          throw new RuntimeException("Replica without node name! " + replica);
        }
        if (rNodeName.equals(nodeName)) {
          log.debug("Update replica state for {} to {}", replica, Replica.State.DOWN);
          Map<String, Object> props = replica.shallowCopy();
          props.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
          Replica newReplica = new Replica(replica.getName(), props, collection, slice.getName());
          newReplicas.put(replica.getName(), newReplica);
          needToUpdateCollection = true;
        }
      }

      Slice newSlice = new Slice(slice.getName(), newReplicas, slice.shallowCopy(),collection);
      slicesCopy.put(slice.getName(), newSlice);
    }

    if (needToUpdateCollection) {
      zkWriteCommands.add(new ZkWriteCommand(collection, docCollection.copyWithSlices(slicesCopy)));
    }
  }

  return zkWriteCommands;
}
 

@Test
public void deleteReplicaFromClusterState() throws Exception {
  final String collectionName = "deleteFromClusterStateCollection";
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 3)
      .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 1, 3);
  
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1"));
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2"));
  cluster.getSolrClient().commit(collectionName);

  cluster.waitForActiveCollection(collectionName, 1, 3);

  Slice shard = getCollectionState(collectionName).getSlice("shard1");

  // don't choose the leader to shutdown, it just complicates things unnecessarily
  Replica replica = getRandomReplica(shard, (r) ->
                                     ( r.getState() == Replica.State.ACTIVE &&
                                       ! r.equals(shard.getLeader())));
  
  JettySolrRunner replicaJetty = cluster.getReplicaJetty(replica);
  ZkStateReaderAccessor accessor = new ZkStateReaderAccessor(replicaJetty.getCoreContainer().getZkController().getZkStateReader());

  final long preDeleteWatcherCount = countUnloadCoreOnDeletedWatchers
    (accessor.getStateWatchers(collectionName));

  ZkNodeProps m = new ZkNodeProps(
      Overseer.QUEUE_OPERATION, OverseerAction.DELETECORE.toLower(),
      ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
      ZkStateReader.NODE_NAME_PROP, replica.getNodeName(),
      ZkStateReader.COLLECTION_PROP, collectionName,
      ZkStateReader.CORE_NODE_NAME_PROP, replica.getName(),
      ZkStateReader.BASE_URL_PROP, replica.getBaseUrl());

  cluster.getOpenOverseer().getStateUpdateQueue().offer(Utils.toJSON(m));

  waitForState("Timeout waiting for replica get deleted", collectionName,
      (liveNodes, collectionState) -> collectionState.getSlice("shard1").getReplicas().size() == 2);

  TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  timeOut.waitFor("Waiting for replica get unloaded", () ->
      replicaJetty.getCoreContainer().getCoreDescriptor(replica.getCoreName()) == null
  );
  
  // the core should no longer have a watch collection state since it was removed
  timeOut = new TimeOut(60, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  timeOut.waitFor("Waiting for core's watcher to be removed", () -> {
      final long postDeleteWatcherCount = countUnloadCoreOnDeletedWatchers
        (accessor.getStateWatchers(collectionName));
      log.info("preDeleteWatcherCount={} vs postDeleteWatcherCount={}",
               preDeleteWatcherCount, postDeleteWatcherCount);
      return (preDeleteWatcherCount - 1L == postDeleteWatcherCount);
    });
  
  CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
}
 

@Test
public void testColStatus() throws Exception {
  final String collectionName = "collectionStatusTest";
  CollectionAdminRequest.createCollection(collectionName, "conf2", 2, 2)
      .process(cluster.getSolrClient());

  cluster.waitForActiveCollection(collectionName, 2, 4);

  SolrClient client = cluster.getSolrClient();
  byte[] binData = collectionName.getBytes("UTF-8");
  // index some docs
  for (int i = 0; i < 10; i++) {
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", String.valueOf(i));
    doc.addField("number_i", i);
    doc.addField("number_l", i);
    doc.addField("number_f", i);
    doc.addField("number_d", i);
    doc.addField("number_ti", i);
    doc.addField("number_tl", i);
    doc.addField("number_tf", i);
    doc.addField("number_td", i);
    doc.addField("point", i + "," + i);
    doc.addField("pointD", i + "," + i);
    doc.addField("store", (i * 5) + "," + (i * 5));
    doc.addField("boolean_b", true);
    doc.addField("multi_int_with_docvals", i);
    doc.addField("string_s", String.valueOf(i));
    doc.addField("tv_mv_string", "this is a test " + i);
    doc.addField("timestamp_dt", new Date());
    doc.addField("timestamp_tdt", new Date());
    doc.addField("payload", binData);
    client.add(collectionName, doc);
  }
  client.commit(collectionName);

  CollectionAdminRequest.ColStatus req = CollectionAdminRequest.collectionStatus(collectionName);
  req.setWithFieldInfo(true);
  req.setWithCoreInfo(true);
  req.setWithSegments(true);
  req.setWithSizeInfo(true);
  CollectionAdminResponse rsp = req.process(cluster.getSolrClient());
  assertEquals(0, rsp.getStatus());
  @SuppressWarnings({"unchecked"})
  List<Object> nonCompliant = (List<Object>)rsp.getResponse().findRecursive(collectionName, "schemaNonCompliant");
  assertEquals(nonCompliant.toString(), 1, nonCompliant.size());
  assertTrue(nonCompliant.toString(), nonCompliant.contains("(NONE)"));
  @SuppressWarnings({"unchecked"})
  NamedList<Object> segInfos = (NamedList<Object>) rsp.getResponse().findRecursive(collectionName, "shards", "shard1", "leader", "segInfos");
  assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("info", "core", "startTime"));
  assertNotNull(Utils.toJSONString(rsp), segInfos.get("fieldInfoLegend"));
  assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("segments", "_0", "fields", "id", "flags"));
  assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("segments", "_0", "ramBytesUsed"));
  // test for replicas not active - SOLR-13882
  DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getClusterState().getCollection(collectionName);
  Replica firstReplica = coll.getSlice("shard1").getReplicas().iterator().next();
  String firstNode = firstReplica.getNodeName();
  for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
    if (jetty.getNodeName().equals(firstNode)) {
      cluster.stopJettySolrRunner(jetty);
    }
  }
  rsp = req.process(cluster.getSolrClient());
  assertEquals(0, rsp.getStatus());
  Number down = (Number) rsp.getResponse().findRecursive(collectionName, "shards", "shard1", "replicas", "down");
  assertTrue("should be some down replicas, but there were none in shard1:" + rsp, down.intValue() > 0);
}
 

private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(String coll, Replica replica, String targetNode, String shardId) {
  return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName());
}