Java Code Examples for org.apache.solr.client.solrj.impl.CloudSolrClient#getZkStateReader()

@Override
protected Map<String, Table> getTableMap() {
  String zk = this.properties.getProperty("zk");
  CloudSolrClient cloudSolrClient = solrClientCache.getCloudSolrClient(zk);
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
  ClusterState clusterState = zkStateReader.getClusterState();

  final ImmutableMap.Builder<String, Table> builder = ImmutableMap.builder();

  Set<String> collections = clusterState.getCollectionsMap().keySet();
  for (String collection : collections) {
    builder.put(collection, new SolrTable(this, collection));
  }

  Aliases aliases = zkStateReader.getAliases();
  for (String alias : aliases.getCollectionAliasListMap().keySet()) {
    // don't create duplicate entries
    if (!collections.contains(alias)) {
      builder.put(alias, new SolrTable(this, alias));
    }
  }

  return builder.build();
}
 

@Before
public void createCollection() throws Exception {

  final String collectionName = testName.getMethodName();
  final CloudSolrClient cloudSolrClient = cluster.getSolrClient();
  
  final Map<String, String> collectionProperties = new HashMap<>();
  collectionProperties.put(CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml");
  
  CollectionAdminRequest.Create cmd = 
    CollectionAdminRequest.createCollection(collectionName, configName,
                                            NUM_SHARDS, REPLICATION_FACTOR)
    .setProperties(collectionProperties);

  if (random().nextBoolean()) {
    assertTrue( cmd.process(cloudSolrClient).isSuccess() );
  } else { // async
    assertEquals(RequestStatusState.COMPLETED, cmd.processAndWait(cloudSolrClient, 30));
  }
  
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
  cluster.waitForActiveCollection(collectionName, NUM_SHARDS, NUM_SHARDS * REPLICATION_FACTOR);
  
  cloudSolrClient.setDefaultCollection(collectionName);
}
 

@SuppressWarnings("unchecked")
private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) {
  ZkStateReader zkStateReader = solrClient.getZkStateReader();
  Collection<Slice> activeSlices = zkStateReader.getClusterState().getCollection(solrClient.getDefaultCollection()).getActiveSlices();
  
  List<LukeResponse> lukeResponses = new ArrayList<>();
  for (Slice slice : activeSlices) {
    for (Replica replica : slice.getReplicas()) {
      try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) {
        HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX);
        HttpResponse response = httpClient.execute(request);
        @SuppressWarnings("resource") // JavaBinCodec implements Closeable, yet it can't be closed if it is used for unmarshalling only
        NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec().unmarshal(response.getEntity().getContent());
        LukeResponse lukeResponse = new LukeResponse();
        lukeResponse.setResponse(lukeData);
        lukeResponses.add(lukeResponse);
      } catch (IOException e) {
        logger.error("Exception during getting luke responses", e);
      }
    }
  }
  return lukeResponses;
}
 

protected void testConfigBackupOnly(String configName, String collectionName) throws Exception {
  String backupName = "configonlybackup";
  CloudSolrClient solrClient = cluster.getSolrClient();

  CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
      .setRepositoryName(getBackupRepoName())
      .setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
  backup.process(solrClient);

  Map<String,String> params = new HashMap<>();
  params.put("location", "/backup");
  params.put("solr.hdfs.home", hdfsUri + "/solr");

  HdfsBackupRepository repo = new HdfsBackupRepository();
  repo.init(new NamedList<>(params));
  BackupManager mgr = new BackupManager(repo, solrClient.getZkStateReader());

  URI baseLoc = repo.createURI("/backup");

  Properties props = mgr.readBackupProperties(baseLoc, backupName);
  assertNotNull(props);
  assertEquals(collectionName, props.getProperty(COLLECTION_NAME_PROP));
  assertEquals(backupName, props.getProperty(BACKUP_NAME_PROP));
  assertEquals(configName, props.getProperty(COLL_CONF));

  DocCollection collectionState = mgr.readCollectionState(baseLoc, backupName, collectionName);
  assertNotNull(collectionState);
  assertEquals(collectionName, collectionState.getName());

  URI configDirLoc = repo.resolve(baseLoc, backupName, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
  assertTrue(repo.exists(configDirLoc));

  Collection<String> expected = Arrays.asList(BACKUP_PROPS_FILE, ZK_STATE_DIR);
  URI backupLoc = repo.resolve(baseLoc, backupName);
  String[] dirs = repo.listAll(backupLoc);
  for (String d : dirs) {
    assertTrue(expected.contains(d));
  }
}
 

/**
 * Wait for all the collection shards to be ready.
 */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;

        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.updateClusterState();
            ClusterState clusterState = zkStateReader.getClusterState();
            Map<String, Slice> slices = clusterState.getSlicesMap(collection);
            Preconditions.checkNotNull("Could not find collection:" + collection, slices);

            for (Map.Entry<String, Slice> entry : slices.entrySet()) {
                Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (Map.Entry<String, Replica> shard : shards.entrySet()) {
                    String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
                    if ((state.equals(Replica.State.RECOVERING.toString())
                            || state.equals(Replica.State.DOWN.toString()))
                            && clusterState.liveNodesContain(shard.getValue().getStr(
                            ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }
            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}
 

/**
 * Checks if the collection has already been created in Solr.
 */
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    zkStateReader.updateClusterState();
    ClusterState clusterState = zkStateReader.getClusterState();
    return clusterState.getCollectionOrNull(collection) != null;
}
 

protected List<String> getAliasList(CloudSolrClient cloudSolrClient, String collectionAlias) {
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
  Aliases aliases = zkStateReader.getAliases();
  String collectionsInAlias = aliases.getCollectionAlias(collectionAlias);
  log.info("Looked up collection list "+collectionsInAlias+" for collection collectionsInAlias: "+collectionAlias);
  return (collectionsInAlias != null) ? StrUtils.splitSmart(collectionsInAlias, ",", true) : new ArrayList<String>(0);
}
 

private void waitForRecoveriesToFinish(String collection, boolean verbose) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ZkStateReader zkStateReader = client.getZkStateReader();
    super.waitForRecoveriesToFinish(collection, zkStateReader, verbose);
  } finally {
    client.close();
  }
}
 

private void waitForCollectionToDisappear(String collection) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ZkStateReader zkStateReader = client.getZkStateReader();
    AbstractDistribZkTestBase.waitForCollectionToDisappear(collection, zkStateReader, true, 15);
  } finally {
    client.close();
  }
}
 

private boolean haveCollection(String alias, String collection) {
  // separated into separate lines to make it easier to track down an NPE that occurred once
  // 3000 runs if it shows up again...
  CloudSolrClient solrClient = cluster.getSolrClient();
  ZkStateReader zkStateReader = solrClient.getZkStateReader();
  Aliases aliases = zkStateReader.getAliases();
  Map<String, List<String>> collectionAliasListMap = aliases.getCollectionAliasListMap();
  List<String> strings = collectionAliasListMap.get(alias);
  return strings.contains(collection);
}
 

@BeforeClass
public static void setupCluster() throws Exception {
  
  differentUniqueId = random().nextBoolean();
  
  final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");

  String configName = "solrCloudCollectionConfig";
  int nodeCount = 5;
  configureCluster(nodeCount)
     .addConfig(configName, configDir)
     .configure();
  
  Map<String, String> collectionProperties = new HashMap<>();
  collectionProperties.put("config", "solrconfig-doctransformers.xml" );
  collectionProperties.put("schema", "schema-docValuesJoin.xml"); 

  int shards = 2;
  int replicas = 2 ;
  CollectionAdminRequest.createCollection(people, configName, shards, replicas)
      .withProperty("config", "solrconfig-doctransformers.xml")
      .withProperty("schema", "schema-docValuesJoin.xml")
      .process(cluster.getSolrClient());

  CollectionAdminRequest.createCollection(depts, configName, shards, replicas)
      .withProperty("config", "solrconfig-doctransformers.xml")
      .withProperty("schema", 
            differentUniqueId ? "schema-minimal-with-another-uniqkey.xml":
                                "schema-docValuesJoin.xml")
      .process(cluster.getSolrClient());

  CloudSolrClient client = cluster.getSolrClient();
  client.setDefaultCollection(people);
  
  ZkStateReader zkStateReader = client.getZkStateReader();
  AbstractDistribZkTestBase.waitForRecoveriesToFinish(people, zkStateReader, true, true, 30);
  
  AbstractDistribZkTestBase.waitForRecoveriesToFinish(depts, zkStateReader, false, true, 30);
}
 

/**
 * Wait for all the collection shards to be ready.
 */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    final ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;

        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.forceUpdateCollection(collection);
            final ClusterState clusterState = zkStateReader.getClusterState();
            final Map<String, Slice> slices = clusterState.getCollection(collection).getSlicesMap();
            Preconditions.checkNotNull(slices, "Could not find collection:" + collection);

            // change paths for Replica.State per Solr refactoring
            // remove SYNC state per: http://tinyurl.com/pag6rwt
            for (final Map.Entry<String, Slice> entry : slices.entrySet()) {
                final Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (final Map.Entry<String, Replica> shard : shards.entrySet()) {
                    final String state = shard.getValue().getStr(ZkStateReader.STATE_PROP).toUpperCase();
                    if ((Replica.State.RECOVERING.name().equals(state) || Replica.State.DOWN.name().equals(state))
                            && clusterState.liveNodesContain(shard.getValue().getStr(
                            ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }


            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}
 

/**
 * Checks if the collection has already been created in Solr.
 */
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    final ZkStateReader zkStateReader = server.getZkStateReader();
    zkStateReader.forceUpdateCollection(collection);
    final ClusterState clusterState = zkStateReader.getClusterState();
    return clusterState.getCollectionOrNull(collection) != null;
}
 

@Override
public boolean exists() throws BackendException {
    if (mode!= Mode.CLOUD) throw new UnsupportedOperationException("Operation only supported for SolrCloud");
    final CloudSolrClient server = (CloudSolrClient) solrClient;
    try {
        final ZkStateReader zkStateReader = server.getZkStateReader();
        zkStateReader.forciblyRefreshAllClusterStateSlow();
        final ClusterState clusterState = zkStateReader.getClusterState();
        final Map<String, DocCollection> collections = clusterState.getCollectionsMap();
        return collections != null && !collections.isEmpty();
    } catch (KeeperException | InterruptedException e) {
        throw new PermanentBackendException("Unable to check if index exists", e);
    }
}
 

@SuppressWarnings({"unchecked"})
public static List<String> getShards(String zkHost,
                                     String collection,
                                     StreamContext streamContext,
                                     SolrParams requestParams)
    throws IOException {
  Map<String, List<String>> shardsMap = null;
  List<String> shards = new ArrayList<>();

  if(streamContext != null) {
    shardsMap = (Map<String, List<String>>)streamContext.get("shards");
  }

  if(shardsMap != null) {
    //Manual Sharding
    shards = shardsMap.get(collection);
  } else {
    //SolrCloud Sharding
    CloudSolrClient cloudSolrClient =
        Optional.ofNullable(streamContext.getSolrClientCache()).orElseGet(SolrClientCache::new).getCloudSolrClient(zkHost);
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    Slice[] slices = CloudSolrStream.getSlices(collection, zkStateReader, true);
    Set<String> liveNodes = clusterState.getLiveNodes();


    ModifiableSolrParams solrParams = new ModifiableSolrParams(streamContext.getRequestParams());
    solrParams.add(requestParams);

    RequestReplicaListTransformerGenerator requestReplicaListTransformerGenerator =
        Optional.ofNullable(streamContext.getRequestReplicaListTransformerGenerator()).orElseGet(RequestReplicaListTransformerGenerator::new);

    ReplicaListTransformer replicaListTransformer = requestReplicaListTransformerGenerator.getReplicaListTransformer(solrParams);

    for(Slice slice : slices) {
      List<Replica> sortedReplicas = new ArrayList<>();
      for(Replica replica : slice.getReplicas()) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
          sortedReplicas.add(replica);
        }
      }

      replicaListTransformer.transform(sortedReplicas);
      if (sortedReplicas.size() > 0) {
        shards.add(sortedReplicas.get(0).getCoreUrl());
      }
    }
  }
  Object core = streamContext.get("core");
  if (streamContext != null && streamContext.isLocal() && core != null) {
    shards.removeIf(shardUrl -> !shardUrl.contains((CharSequence) core));
  }

  return shards;
}
 

/**
 * Updates the mappings between the jetty's instances and the zookeeper cluster state.
 */
protected void updateMappingsFromZk(String collection) throws Exception {
  List<CloudJettyRunner> cloudJettys = new ArrayList<>();
  Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
  Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();

  CloudSolrClient cloudClient = this.createCloudClient(null);
  try {
    cloudClient.connect();
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(collection);

    for (JettySolrRunner jetty : jettys) {
      int port = jetty.getLocalPort();
      if (port == -1) {
        throw new RuntimeException("Cannot find the port for jetty");
      }

      nextJetty:
      for (Slice shard : coll.getSlices()) {
        Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
        for (Map.Entry<String, Replica> entry : entries) {
          Replica replica = entry.getValue();
          if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
            if (!shardToJetty.containsKey(shard.getName())) {
              shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
            }
            boolean isLeader = shard.getLeader() == replica;
            CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
            shardToJetty.get(shard.getName()).add(cjr);
            if (isLeader) {
              shardToLeaderJetty.put(shard.getName(), cjr);
            }
            cloudJettys.add(cjr);
            break nextJetty;
          }
        }
      }
    }

    List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
    if (oldRunners != null)  {
      // must close resources for the old entries
      for (CloudJettyRunner oldRunner : oldRunners) {
        IOUtils.closeQuietly(oldRunner.client);
      }
    }

    this.cloudJettys.put(collection, cloudJettys);
    this.shardToJetty.put(collection, shardToJetty);
    this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
  } finally {
    cloudClient.close();
  }
}
 

/**
 * Returns the list of shards of the default collection.
 *
 * @param zkHost            ZooKeeper URL
 * @param chronixCollection Solr collection name for chronix time series data
 * @return the list of shards of the default collection
 */
public List<String> getShardList(String zkHost, String chronixCollection) throws IOException {

    CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost);
    List<String> shards = new ArrayList<>();

    try {
        cloudSolrClient.connect();

        ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

        ClusterState clusterState = zkStateReader.getClusterState();

        String[] collections;
        if (clusterState.hasCollection(chronixCollection)) {
            collections = new String[]{chronixCollection};
        } else {
            // might be a collection alias?
            Aliases aliases = zkStateReader.getAliases();
            String aliasedCollections = aliases.getCollectionAlias(chronixCollection);
            if (aliasedCollections == null)
                throw new IllegalArgumentException("Collection " + chronixCollection + " not found!");
            collections = aliasedCollections.split(",");
        }

        Set<String> liveNodes = clusterState.getLiveNodes();
        Random random = new Random(5150);

        for (String coll : collections) {
            for (Slice slice : clusterState.getSlices(coll)) {
                List<String> replicas = new ArrayList<>();
                for (Replica r : slice.getReplicas()) {
                    if (r.getState().equals(Replica.State.ACTIVE)) {
                        ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
                        if (liveNodes.contains(replicaCoreProps.getNodeName()))
                            replicas.add(replicaCoreProps.getCoreUrl());
                    }
                }
                int numReplicas = replicas.size();
                if (numReplicas == 0)
                    throw new IllegalStateException("Shard " + slice.getName() + " in collection " +
                            coll + " does not have any active replicas!");

                String replicaUrl = (numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
                shards.add(replicaUrl);
            }
        }
    } finally {
        cloudSolrClient.close();
    }

    return shards;
}
 

private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
  ZkStateReader reader = solrClient.getZkStateReader();
  DocCollection collection = reader.getClusterState().getCollection(solrPropsConfig.getCollection());
  return collection.getSlices();
}