Java Code Examples for org.apache.solr.common.cloud.DocCollection#getReplicas()

protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
  ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection collection = clusterState.getCollectionOrNull(collectionName, true);
  if (collection == null) {
    return null;
  }

  Set<String> liveNodes = clusterState.getLiveNodes();

  if (isPreferLeader) {
    List<Replica> leaderReplicas = collection.getLeaderReplicas(cores.getZkController().getNodeName());
    SolrCore core = randomlyGetSolrCore(liveNodes, leaderReplicas);
    if (core != null) return core;
  }

  List<Replica> replicas = collection.getReplicas(cores.getZkController().getNodeName());
  return randomlyGetSolrCore(liveNodes, replicas);
}
 

@Test
public void testCreateNodeSet() throws Exception {
  JettySolrRunner jetty1 = cluster.getRandomJetty(random());
  JettySolrRunner jetty2 = cluster.getRandomJetty(random());

  List<String> baseUrls = ImmutableList.of(jetty1.getBaseUrl().toString(), jetty2.getBaseUrl().toString());

  CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1)
      .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1))
      .process(cluster.getSolrClient());

  DocCollection collectionState = getCollectionState("nodeset_collection");
  for (Replica replica : collectionState.getReplicas()) {
    String replicaUrl = replica.getCoreUrl();
    boolean matchingJetty = false;
    for (String jettyUrl : baseUrls) {
      if (replicaUrl.startsWith(jettyUrl)) {
        matchingJetty = true;
      }
    }
    if (matchingJetty == false) {
      fail("Expected replica to be on " + baseUrls + " but was on " + replicaUrl);
    }
  }
}
 

@Test
public void queryCollections() throws Exception {
  List<Collector.MetricFamilySamples> collection1Metrics = solrCloudScraper.collections(
      configuration.getCollectionsConfiguration().get(0)).asList();

  assertEquals(2, collection1Metrics.size());
  Collector.MetricFamilySamples liveNodeSamples = collection1Metrics.get(0);
  assertEquals("solr_collections_live_nodes", liveNodeSamples.name);
  assertEquals("See following URL: https://lucene.apache.org/solr/guide/collections-api.html#clusterstatus", liveNodeSamples.help);
  assertEquals(1, liveNodeSamples.samples.size());

  assertEquals(
      getClusterState().getLiveNodes().size(),
      liveNodeSamples.samples.get(0).value, 0.001);

  Collector.MetricFamilySamples shardLeaderSamples = collection1Metrics.get(1);

  DocCollection collection = getCollectionState();
  List<Replica> allReplicas = collection.getReplicas();
  assertEquals(allReplicas.size(), shardLeaderSamples.samples.size());

  Collection<Slice> slices = getCollectionState().getSlices();

  Set<String> leaderCoreNames = slices.stream()
      .map(slice -> collection.getLeader(slice.getName()).getCoreName())
      .collect(Collectors.toSet());

  for (Collector.MetricFamilySamples.Sample sample : shardLeaderSamples.samples) {
    assertEquals("solr_collections_shard_leader", sample.name);
    assertEquals(Arrays.asList("collection", "shard", "replica", "core", "type", "zk_host"), sample.labelNames);
    assertEquals(leaderCoreNames.contains(sample.labelValues.get(3)) ? 1.0 : 0.0, sample.value, 0.001);
  }
}
 

private void checkSharedFsReplicasMovedCorrectly(List<Replica> replacedHdfsReplicas, ZkStateReader zkStateReader, String collection){
  DocCollection docCollection = zkStateReader.getClusterState().getCollection(collection);
  for (Replica replica :replacedHdfsReplicas) {
    boolean found = false;
    String dataDir = replica.getStr("dataDir");
    String ulogDir = replica.getStr("ulogDir");
    for (Replica replica2 : docCollection.getReplicas()) {
      if (dataDir.equals(replica2.getStr("dataDir")) && ulogDir.equals(replica2.getStr("ulogDir"))) {
        found = true;
        break;
      }
    }
    if (!found) fail("Can not found a replica with same dataDir and ulogDir as " + replica + " from:" + docCollection.getReplicas());
  }
}
 

/**
 * returns a map whose key is the coreNodeName and whose value is data about that core needed for the test
 */
private Map<String,ReplicaData> getTestDataForAllReplicas() throws IOException, SolrServerException {
  Map<String,ReplicaData> results = new HashMap<>();

  DocCollection collectionState = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION);

  for (Replica replica : collectionState.getReplicas()) {

    String coreName = replica.getCoreName();
    try (HttpSolrClient client = getHttpSolrClient(replica.getCoreUrl())) {

      ModifiableSolrParams params = new ModifiableSolrParams();
      params.set("command", "indexversion");
      params.set("_trace", "getIndexVersion");
      params.set("qt", ReplicationHandler.PATH);
      QueryRequest req = setAuthIfNeeded(new QueryRequest(params));

      NamedList<Object> res = client.request(req);
      assertNotNull("null response from server: " + coreName, res);

      Object version = res.get("indexversion");
      assertNotNull("null version from server: " + coreName, version);
      assertTrue("version isn't a long: " + coreName, version instanceof Long);

      long numDocs = 
        setAuthIfNeeded(new QueryRequest
                        (params("q", "*:*",
                                "distrib", "false",
                                "rows", "0",
                                "_trace", "counting_docs"))).process(client).getResults().getNumFound();

      final ReplicaData data = new ReplicaData(replica.getSlice(),coreName,(Long)version,numDocs);
      log.info("{}", data);
      results.put(coreName, data);

    }
  }

  return results;
}
 

private String getDaemonUrl(SolrResponse rsp, DocCollection coll) {
  @SuppressWarnings({"unchecked"})
  Map<String, Object> rs = (Map<String, Object>)rsp.getResponse().get("result-set");
  if (rs == null || rs.isEmpty()) {
    if (log.isDebugEnabled()) {
      log.debug(" -- Missing daemon information in response: {}", Utils.toJSONString(rsp));
    }
  }
  @SuppressWarnings({"unchecked"})
  List<Object> list = (List<Object>)rs.get("docs");
  if (list == null) {
    if (log.isDebugEnabled()) {
      log.debug(" -- Missing daemon information in response: {}", Utils.toJSONString(rsp));
    }
    return null;
  }
  String replicaName = null;
  for (Object o : list) {
    @SuppressWarnings({"unchecked"})
    Map<String, Object> map = (Map<String, Object>)o;
    String op = (String)map.get("DaemonOp");
    if (op == null) {
      continue;
    }
    String[] parts = op.split("\\s+");
    if (parts.length != 4) {
      log.debug(" -- Invalid daemon location info, expected 4 tokens: {}", op);
      return null;
    }
    // check if it's plausible
    if (parts[3].contains("shard") && parts[3].contains("replica")) {
      replicaName = parts[3];
      break;
    } else {
      log.debug(" -- daemon location info likely invalid: {}", op);
      return null;
    }
  }
  if (replicaName == null) {
    return null;
  }
  // build a baseUrl of the replica
  for (Replica r : coll.getReplicas()) {
    if (replicaName.equals(r.getCoreName())) {
      return r.getBaseUrl() + "/" + r.getCoreName();
    }
  }
  return null;
}
 

private static void waitForRecovery(String collection) throws Exception {
  log.info("Waiting for recovery of {}", collection);
  boolean recovered = false;
  boolean allActive = true;
  boolean hasLeaders = true;
  DocCollection collState = null;
  for (int i = 0; i < 300; i++) {
    ClusterState state = solrClient.getZkStateReader().getClusterState();
    collState = getCollectionState(collection);
    log.debug("###### {}", collState);
    Collection<Replica> replicas = collState.getReplicas();
    allActive = true;
    hasLeaders = true;
    if (replicas != null && !replicas.isEmpty()) {
      for (Replica r : replicas) {
        if (state.getLiveNodes().contains(r.getNodeName())) {
          if (!r.isActive(state.getLiveNodes())) {
            log.info("Not active: {}", r);
            allActive = false;
          }
        } else {
          log.info("Replica no longer on a live node, ignoring: {}", r);
        }
      }
    } else {
      allActive = false;
    }
    for (Slice slice : collState.getSlices()) {
      if (slice.getLeader() == null) {
        hasLeaders = false;
      }
    }
    if (allActive && hasLeaders) {
      recovered = true;
      break;
    } else {
      log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders);
      Thread.sleep(1000);
    }
  }
  assertTrue("replica never fully recovered: allActive=" + allActive + ", hasLeaders=" + hasLeaders + ", collState=" + collState, recovered);

}
 

@Test
  // commented out on: 24-Dec-2018   @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018
  public void testExecute() throws Exception {
    SolrClient solrClient = cluster.simGetSolrClient();
    String collectionName = "testExecute";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
        "conf", 1, 2);
    create.setMaxShardsPerNode(1);
    create.process(solrClient);

    if (log.isInfoEnabled()) {
      log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 120, TimeUnit.SECONDS,
          CloudUtil.clusterShape(1, 2, false, true)));
    }

    String sourceNodeName = cluster.getSimClusterStateProvider().simGetRandomNode();
    ClusterState clusterState = cluster.getClusterStateProvider().getClusterState();
    DocCollection docCollection = clusterState.getCollection(collectionName);
    List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
    assertNotNull(replicas);
    assertFalse(replicas.isEmpty());

    List<String> otherNodes = cluster.getClusterStateProvider().getLiveNodes().stream()
        .filter(node -> !node.equals(sourceNodeName)).collect(Collectors.toList());
    assertFalse(otherNodes.isEmpty());
    String survivor = otherNodes.get(0);

    try (ExecutePlanAction action = new ExecutePlanAction()) {
      action.configure(cluster.getLoader(), cluster, Collections.singletonMap("name", "execute_plan"));

      // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
      AtomicBoolean znodeCreated = new AtomicBoolean(false);

      CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor);
      CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
        @Override
        public void setAsyncId(String asyncId) {
          super.setAsyncId(asyncId);
          String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
          try {
            if (cluster.getDistribStateManager().hasData(parentPath)) {
              java.util.List<String> children = cluster.getDistribStateManager().listData(parentPath);
              if (!children.isEmpty()) {
                String child = children.get(0);
                VersionedData data = cluster.getDistribStateManager().getData(parentPath + "/" + child);
                @SuppressWarnings({"rawtypes"})
                Map m = (Map) Utils.fromJSON(data.getData());
                if (m.containsKey("requestid")) {
                  znodeCreated.set(m.get("requestid").equals(asyncId));
                }
              }
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }

        }
      };
      List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
      NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST,
          "mock_trigger_name", Collections.singletonList(SIM_TIME_SOURCE.getTimeNs()),
          Collections.singletonList(sourceNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
      ActionContext actionContext = new ActionContext(cluster, null,
          new HashMap<>(Collections.singletonMap("operations", operations)));
      action.process(nodeLostEvent, actionContext);

//      assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
      @SuppressWarnings({"unchecked"})
      List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
      assertNotNull(responses);
      assertEquals(2, responses.size());
      NamedList<Object> response = responses.get(0);
      assertNull(response.get("failure"));
      assertNotNull(response.get("success"));
    }

    if (log.isInfoEnabled()) {
      log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 300, TimeUnit.SECONDS,
          CloudUtil.clusterShape(1, 2, false, true)));
    }
  }
 

public void testCreateCollectionSplitShard() throws Exception  {
  SolrClient solrClient = cluster.simGetSolrClient();
  String firstNode = cluster.getSimClusterStateProvider().simGetRandomNode();
  int firstNodePort = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(firstNode, ImplicitSnitch.PORT);

  String secondNode;
  int secondNodePort;
  while (true)  {
    secondNode = cluster.getSimClusterStateProvider().simGetRandomNode();
    secondNodePort = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(secondNode, ImplicitSnitch.PORT);
    if (secondNodePort != firstNodePort)  break;
  }

  String commands =  "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" + firstNodePort + "'}, {replica:1, shard:'#EACH', port:'" + secondNodePort + "'}]}}";
  NamedList<Object> response = solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
  assertEquals("success", response.get("result"));

  String collectionName = "testCreateCollectionSplitShard";
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2)
      .setPolicy("c1")
      .process(solrClient);
  CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", collectionName,
      CloudUtil.clusterShape(1, 2, false, true));

  DocCollection docCollection = getCollectionState(collectionName);
  List<Replica> list = docCollection.getReplicas(firstNode);
  int replicasOnNode1 = list != null ? list.size() : 0;
  list = docCollection.getReplicas(secondNode);
  int replicasOnNode2 = list != null ? list.size() : 0;

  assertEquals("Expected exactly one replica of collection on node with port: " + firstNodePort, 1, replicasOnNode1);
  assertEquals("Expected exactly one replica of collection on node with port: " + secondNodePort, 1, replicasOnNode2);

  CollectionAdminRequest.splitShard(collectionName).setShardName("shard1").process(solrClient);

  CloudUtil.waitForState(cluster, "Timed out waiting to see 6 replicas for collection: " + collectionName,
      collectionName, (liveNodes, collectionState) -> collectionState.getReplicas().size() == 6);

  docCollection = getCollectionState(collectionName);
  list = docCollection.getReplicas(firstNode);
  replicasOnNode1 = list != null ? list.size() : 0;
  list = docCollection.getReplicas(secondNode);
  replicasOnNode2 = list != null ? list.size() : 0;

  assertEquals("Expected exactly three replica of collection on node with port: " + firstNodePort, 3, replicasOnNode1);
  assertEquals("Expected exactly three replica of collection on node with port: " + secondNodePort, 3, replicasOnNode2);
  CollectionAdminRequest.deleteCollection(collectionName).process(solrClient);

}
 

@Test
  public void testExecute() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "testExecute";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
        "conf", 1, 2);
    create.setMaxShardsPerNode(1);
    create.process(solrClient);
    
    cluster.waitForActiveCollection(collectionName, 1, 2);

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));

    JettySolrRunner sourceNode = cluster.getRandomJetty(random());
    String sourceNodeName = sourceNode.getNodeName();
    ClusterState clusterState = solrClient.getZkStateReader().getClusterState();
    DocCollection docCollection = clusterState.getCollection(collectionName);
    List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
    assertNotNull(replicas);
    assertFalse(replicas.isEmpty());

    List<JettySolrRunner> otherJetties = cluster.getJettySolrRunners().stream()
        .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList());
    assertFalse(otherJetties.isEmpty());
    JettySolrRunner survivor = otherJetties.get(0);

    try (ExecutePlanAction action = new ExecutePlanAction()) {
      action.configure(loader, cloudManager, Collections.singletonMap("name", "execute_plan"));

      // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
      AtomicBoolean znodeCreated = new AtomicBoolean(false);

      CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
      CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
        @Override
        public void setAsyncId(String asyncId) {
          super.setAsyncId(asyncId);
          String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
          try {
            if (zkClient().exists(parentPath, true)) {
              java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
              if (!children.isEmpty()) {
                String child = children.get(0);
                byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true);
                @SuppressWarnings({"rawtypes"})
                Map m = (Map) Utils.fromJSON(data);
                if (m.containsKey("requestid")) {
                  znodeCreated.set(m.get("requestid").equals(asyncId));
                }
              }
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }

        }
      };
      List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
      NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent
        (TriggerEventType.NODELOST, "mock_trigger_name",
         Collections.singletonList(cloudManager.getTimeSource().getTimeNs()),
         Collections.singletonList(sourceNodeName),
         CollectionParams.CollectionAction.MOVEREPLICA.toLower());
      ActionContext actionContext = new ActionContext(survivor.getCoreContainer().getZkController().getSolrCloudManager(), null,
          new HashMap<>(Collections.singletonMap("operations", operations)));
      action.process(nodeLostEvent, actionContext);

//      assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
      @SuppressWarnings({"unchecked"})
      List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
      assertNotNull(responses);
      assertEquals(2, responses.size());
      NamedList<Object> response = responses.get(0);
      assertNull(response.get("failure"));
      assertNotNull(response.get("success"));
    }

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));
  }
 

@Test
public void test() throws Exception {
  CloudSolrClient cloudClient = cluster.getSolrClient();
  String coll = "deletenodetest_coll";
  ClusterState state = cloudClient.getZkStateReader().getClusterState();
  Set<String> liveNodes = state.getLiveNodes();
  ArrayList<String> l = new ArrayList<>(liveNodes);
  Collections.shuffle(l, random());
  CollectionAdminRequest.Create create = pickRandom(
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1),
      // check RF=1
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 0)
      );
  create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3);
  cloudClient.request(create);
  state = cloudClient.getZkStateReader().getClusterState();
  String node2bdecommissioned = l.get(0);
  // check what replicas are on the node, and whether the call should fail
  boolean shouldFail = false;
  DocCollection docColl = state.getCollection(coll);
  log.info("#### DocCollection: {}", docColl);
  List<Replica> replicas = docColl.getReplicas(node2bdecommissioned);
  if (replicas != null) {
    for (Replica replica : replicas) {
      String shard = docColl.getShardId(node2bdecommissioned, replica.getStr(ZkStateReader.CORE_NAME_PROP));
      Slice slice = docColl.getSlice(shard);
      boolean hasOtherNonPullReplicas = false;
      for (Replica r: slice.getReplicas()) {
        if (!r.getName().equals(replica.getName()) &&
            !r.getNodeName().equals(node2bdecommissioned) &&
            r.getType() != Replica.Type.PULL) {
          hasOtherNonPullReplicas = true;
          break;
        }
      }
      if (!hasOtherNonPullReplicas) {
        shouldFail = true;
        break;
      }
    }
  }
  new CollectionAdminRequest.DeleteNode(node2bdecommissioned).processAsync("003", cloudClient);
  CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("003");
  CollectionAdminRequest.RequestStatusResponse rsp = null;
  for (int i = 0; i < 200; i++) {
    rsp = requestStatus.process(cloudClient);
    if (rsp.getRequestStatus() == RequestStatusState.FAILED || rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
      break;
    }
    Thread.sleep(50);
  }
  if (log.isInfoEnabled()) {
    log.info("####### DocCollection after: {}", cloudClient.getZkStateReader().getClusterState().getCollection(coll));
  }
  if (shouldFail) {
    assertTrue(String.valueOf(rsp), rsp.getRequestStatus() == RequestStatusState.FAILED);
  } else {
    assertFalse(String.valueOf(rsp), rsp.getRequestStatus() == RequestStatusState.FAILED);
  }
}