Java Code Examples for org.apache.solr.common.cloud.Replica#getCoreUrl()

@SuppressWarnings("unchecked")
private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) {
  ZkStateReader zkStateReader = solrClient.getZkStateReader();
  Collection<Slice> activeSlices = zkStateReader.getClusterState().getCollection(solrClient.getDefaultCollection()).getActiveSlices();
  
  List<LukeResponse> lukeResponses = new ArrayList<>();
  for (Slice slice : activeSlices) {
    for (Replica replica : slice.getReplicas()) {
      try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) {
        HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX);
        HttpResponse response = httpClient.execute(request);
        @SuppressWarnings("resource") // JavaBinCodec implements Closeable, yet it can't be closed if it is used for unmarshalling only
        NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec().unmarshal(response.getEntity().getContent());
        LukeResponse lukeResponse = new LukeResponse();
        lukeResponse.setResponse(lukeData);
        lukeResponses.add(lukeResponse);
      } catch (IOException e) {
        logger.error("Exception during getting luke responses", e);
      }
    }
  }
  return lukeResponses;
}
 

public static Map<String, String> mapReplicasToReplicaType(DocCollection collection) {
  Map<String, String> replicaTypeMap = new HashMap<>();
  for (Slice slice : collection.getSlices()) {
    for (Replica replica : slice.getReplicas()) {
      String coreUrl = replica.getCoreUrl();
      // It seems replica reports its core URL with a trailing slash while shard
      // info returned from the query doesn't. Oh well. We will include both, just in case
      replicaTypeMap.put(coreUrl, replica.getType().toString());
      if (coreUrl.endsWith("/")) {
        replicaTypeMap.put(coreUrl.substring(0, coreUrl.length() - 1), replica.getType().toString());
      }else {
        replicaTypeMap.put(coreUrl + "/", replica.getType().toString());
      }
    }
  }
  return replicaTypeMap;
}
 

@Test
public void testCreateNodeSet() throws Exception {
  JettySolrRunner jetty1 = cluster.getRandomJetty(random());
  JettySolrRunner jetty2 = cluster.getRandomJetty(random());

  List<String> baseUrls = ImmutableList.of(jetty1.getBaseUrl().toString(), jetty2.getBaseUrl().toString());

  CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1)
      .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1))
      .process(cluster.getSolrClient());

  DocCollection collectionState = getCollectionState("nodeset_collection");
  for (Replica replica : collectionState.getReplicas()) {
    String replicaUrl = replica.getCoreUrl();
    boolean matchingJetty = false;
    for (String jettyUrl : baseUrls) {
      if (replicaUrl.startsWith(jettyUrl)) {
        matchingJetty = true;
      }
    }
    if (matchingJetty == false) {
      fail("Expected replica to be on " + baseUrls + " but was on " + replicaUrl);
    }
  }
}
 

protected void sendCommitWithRetry(Replica replica) throws Exception {
  String replicaCoreUrl = replica.getCoreUrl();
  log.info("Sending commit request to: {}", replicaCoreUrl);
  final RTimer timer = new RTimer();
  try (HttpSolrClient client = getHttpSolrClient(replicaCoreUrl)) {
    try {
      client.commit();

      if (log.isInfoEnabled()) {
        log.info("Sent commit request to {} OK, took {}ms", replicaCoreUrl, timer.getTime());
      }
    } catch (Exception exc) {
      Throwable rootCause = SolrException.getRootCause(exc);
      if (rootCause instanceof NoHttpResponseException) {
        log.warn("No HTTP response from sending commit request to {}; will re-try after waiting 3 seconds", replicaCoreUrl);
        Thread.sleep(3000);
        client.commit();
        log.info("Second attempt at sending commit to {} succeeded", replicaCoreUrl);
      } else {
        throw exc;
      }
    }
  }
}
 

private long getCheckpoint(Slice slice, Set<String> liveNodes) throws IOException {
  Collection<Replica> replicas = slice.getReplicas();
  long checkpoint = -1;
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set("q","*:*");
  params.set(SORT, "_version_ desc");
  params.set(DISTRIB, "false");
  params.set("rows", 1);
  for(Replica replica : replicas) {
    if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
      String coreUrl = replica.getCoreUrl();
      SolrStream solrStream = new SolrStream(coreUrl, params);

      if(streamContext != null) {
        StreamContext localContext = new StreamContext();
        localContext.setSolrClientCache(streamContext.getSolrClientCache());
        localContext.setObjectCache(streamContext.getObjectCache());
        solrStream.setStreamContext(localContext);
      }

      try {
        solrStream.open();
        Tuple tuple = solrStream.read();
        if(tuple.EOF) {
          return 0;
        } else {
          checkpoint = tuple.getLong("_version_");
        }
        break;
      } finally {
        solrStream.close();
      }
    }
  }
  return checkpoint;
}
 

/**
 * Return the jetty that a particular replica resides on
 */
public JettySolrRunner getReplicaJetty(Replica replica) {
  for (JettySolrRunner jetty : jettys) {
    if (jetty.isStopped()) continue;
    if (replica.getCoreUrl().startsWith(jetty.getBaseUrl().toString()))
      return jetty;
  }
  throw new IllegalArgumentException("Cannot find Jetty for a replica with core url " + replica.getCoreUrl());
}
 

private String getCoreUrl(String collectionName,
                          String origCorename, ClusterState clusterState, List<Slice> slices,
                          boolean byCoreName, boolean activeReplicas) {
  String coreUrl;
  Set<String> liveNodes = clusterState.getLiveNodes();
  Collections.shuffle(slices, random);

  for (Slice slice : slices) {
    List<Replica> randomizedReplicas = new ArrayList<>(slice.getReplicas());
    Collections.shuffle(randomizedReplicas, random);

    for (Replica replica : randomizedReplicas) {
      if (!activeReplicas || (liveNodes.contains(replica.getNodeName())
          && replica.getState() == Replica.State.ACTIVE)) {

        if (byCoreName && !origCorename.equals(replica.getStr(CORE_NAME_PROP))) {
          // if it's by core name, make sure they match
          continue;
        }
        if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
          // don't count a local core
          continue;
        }

        if (origCorename != null) {
          coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
        } else {
          coreUrl = replica.getCoreUrl();
          if (coreUrl.endsWith("/")) {
            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
          }
        }

        return coreUrl;
      }
    }
  }
  return null;
}
 

private BootstrapStatus getBoostrapStatus() throws InterruptedException {
  try {
    Replica leader = state.getClient().getZkStateReader().getLeaderRetry(targetCollection, shard, 30000); // assume same shard exists on target
    String leaderCoreUrl = leader.getCoreUrl();
    HttpClient httpClient = state.getClient().getLbClient().getHttpClient();
    try (HttpSolrClient client = new HttpSolrClient.Builder(leaderCoreUrl).withHttpClient(httpClient).build()) {
      @SuppressWarnings({"rawtypes"})
      NamedList response = sendCdcrCommand(client, CdcrParams.CdcrAction.BOOTSTRAP_STATUS);
      String status = (String) response.get(RESPONSE_STATUS);
      BootstrapStatus bootstrapStatus = BootstrapStatus.valueOf(status.toUpperCase(Locale.ROOT));
      if (bootstrapStatus == BootstrapStatus.RUNNING) {
        return BootstrapStatus.RUNNING;
      } else if (bootstrapStatus == BootstrapStatus.COMPLETED) {
        return BootstrapStatus.COMPLETED;
      } else if (bootstrapStatus == BootstrapStatus.FAILED) {
        return BootstrapStatus.FAILED;
      } else if (bootstrapStatus == BootstrapStatus.NOTFOUND) {
        log.warn("Bootstrap process was not found on target collection: {} shard: {}, leader: {}", targetCollection, shard, leaderCoreUrl);
        return BootstrapStatus.NOTFOUND;
      } else if (bootstrapStatus == BootstrapStatus.CANCELLED) {
        return BootstrapStatus.CANCELLED;
      } else {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
            "Unknown status: " + status + " returned by BOOTSTRAP_STATUS command");
      }
    }
  } catch (Exception e) {
    log.error("Exception during bootstrap status request", e);
    return BootstrapStatus.UNKNOWN;
  }
}
 

private void queryWithPreferReplicaTypes(CloudSolrClient cloudClient,
                                         String preferReplicaTypes,
                                         boolean preferLocalShards,
                                         String collectionName)
    throws Exception
{
  SolrQuery qRequest = new SolrQuery("*:*");
  ModifiableSolrParams qParams = new ModifiableSolrParams();

  final List<String> preferredTypes = Arrays.asList(preferReplicaTypes.split("\\|"));
  StringBuilder rule = new StringBuilder();
  preferredTypes.forEach(type -> {
    if (rule.length() != 0) {
      rule.append(',');
    }
    rule.append(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE);
    rule.append(':');
    rule.append(type);
  });
  if (preferLocalShards) {
    if (rule.length() != 0) {
      rule.append(',');
    }
    rule.append(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION);
    rule.append(":local");
  }
  qParams.add(ShardParams.SHARDS_PREFERENCE, rule.toString());  
  qParams.add(ShardParams.SHARDS_INFO, "true");
  qRequest.add(qParams);

  // CloudSolrClient sends the request to some node.
  // And since all the nodes are hosting cores from all shards, the
  // distributed query formed by this node will select cores from the
  // local shards only
  QueryResponse qResponse = cloudClient.query(collectionName, qRequest);

  Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("Unable to obtain "+ShardParams.SHARDS_INFO, shardsInfo);

  Map<String, String> replicaTypeMap = new HashMap<String, String>();
  DocCollection collection = getCollectionState(collectionName);
  for (Slice slice : collection.getSlices()) {
    for (Replica replica : slice.getReplicas()) {
      String coreUrl = replica.getCoreUrl();
      // It seems replica reports its core URL with a trailing slash while shard
      // info returned from the query doesn't. Oh well.
      if (coreUrl.endsWith("/")) {
        coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
      }
      replicaTypeMap.put(coreUrl, replica.getType().toString());
    }
  }

  // Iterate over shards-info and check that replicas of correct type responded
  SimpleOrderedMap<?> shardsInfoMap = (SimpleOrderedMap<?>)shardsInfo;
  @SuppressWarnings({"unchecked"})
  Iterator<Map.Entry<String, ?>> itr = shardsInfoMap.asMap(100).entrySet().iterator();
  List<String> shardAddresses = new ArrayList<String>();
  while (itr.hasNext()) {
    Map.Entry<String, ?> e = itr.next();
    assertTrue("Did not find map-type value in "+ShardParams.SHARDS_INFO, e.getValue() instanceof Map);
    String shardAddress = (String)((Map)e.getValue()).get("shardAddress");
    assertNotNull(ShardParams.SHARDS_INFO+" did not return 'shardAddress' parameter", shardAddress);
    assertTrue(replicaTypeMap.containsKey(shardAddress));
    assertTrue(preferredTypes.indexOf(replicaTypeMap.get(shardAddress)) == 0);
    shardAddresses.add(shardAddress);
  }
  assertTrue("No responses", shardAddresses.size() > 0);
  if (log.isInfoEnabled()) {
    log.info("Shards giving the response: {}", Arrays.toString(shardAddresses.toArray()));
  }
}
 

private void queryWithPreferReplicaTypes(CloudHttp2SolrClient cloudClient,
                                         String preferReplicaTypes,
                                         boolean preferLocalShards,
                                         String collectionName)
    throws Exception
{
  SolrQuery qRequest = new SolrQuery("*:*");
  ModifiableSolrParams qParams = new ModifiableSolrParams();

  final List<String> preferredTypes = Arrays.asList(preferReplicaTypes.split("\\|"));
  StringBuilder rule = new StringBuilder();
  preferredTypes.forEach(type -> {
    if (rule.length() != 0) {
      rule.append(',');
    }
    rule.append(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE);
    rule.append(':');
    rule.append(type);
  });
  if (preferLocalShards) {
    if (rule.length() != 0) {
      rule.append(',');
    }
    rule.append(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION);
    rule.append(":local");
  }
  qParams.add(ShardParams.SHARDS_PREFERENCE, rule.toString());  
  qParams.add(ShardParams.SHARDS_INFO, "true");
  qRequest.add(qParams);

  // CloudSolrClient sends the request to some node.
  // And since all the nodes are hosting cores from all shards, the
  // distributed query formed by this node will select cores from the
  // local shards only
  QueryResponse qResponse = cloudClient.query(collectionName, qRequest);

  Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("Unable to obtain "+ShardParams.SHARDS_INFO, shardsInfo);

  Map<String, String> replicaTypeMap = new HashMap<>();
  DocCollection collection = getCollectionState(collectionName);
  for (Slice slice : collection.getSlices()) {
    for (Replica replica : slice.getReplicas()) {
      String coreUrl = replica.getCoreUrl();
      // It seems replica reports its core URL with a trailing slash while shard
      // info returned from the query doesn't. Oh well.
      if (coreUrl.endsWith("/")) {
        coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
      }
      replicaTypeMap.put(coreUrl, replica.getType().toString());
    }
  }

  // Iterate over shards-info and check that replicas of correct type responded
  SimpleOrderedMap<?> shardsInfoMap = (SimpleOrderedMap<?>)shardsInfo;
  @SuppressWarnings({"unchecked"})
  Iterator<Map.Entry<String, ?>> itr = shardsInfoMap.asMap(100).entrySet().iterator();
  List<String> shardAddresses = new ArrayList<String>();
  while (itr.hasNext()) {
    Map.Entry<String, ?> e = itr.next();
    assertTrue("Did not find map-type value in "+ShardParams.SHARDS_INFO, e.getValue() instanceof Map);
    String shardAddress = (String)((Map)e.getValue()).get("shardAddress");
    assertNotNull(ShardParams.SHARDS_INFO+" did not return 'shardAddress' parameter", shardAddress);
    assertTrue(replicaTypeMap.containsKey(shardAddress));
    assertTrue(preferredTypes.indexOf(replicaTypeMap.get(shardAddress)) == 0);
    shardAddresses.add(shardAddress);
  }
  assertTrue("No responses", shardAddresses.size() > 0);
  if (log.isInfoEnabled()) {
    log.info("Shards giving the response: {}", Arrays.toString(shardAddresses.toArray()));
  }
}
 

static ReplicaInfo create(Replica replica) {
  return new AutoValue_SolrIO_ReplicaInfo(
      replica.getStr(ZkStateReader.CORE_NAME_PROP),
      replica.getCoreUrl(),
      replica.getStr(ZkStateReader.BASE_URL_PROP));
}