Java Code Examples for org.apache.solr.common.cloud.SolrZkClient#getChildren()

/**
 * Create a merged view of all collections from /collections/?/state.json
 */
private synchronized List<String> getCollections(SolrZkClient zkClient) throws KeeperException, InterruptedException {
  if (cachedCollections == null) {
    // cache is stale, rebuild the full list ...
    cachedCollections = new ArrayList<String>();

    List<String> fromZk = zkClient.getChildren("/collections", this, true);
    if (fromZk != null)
      cachedCollections.addAll(fromZk);

    // sort the final merged set of collections
    Collections.sort(cachedCollections, this);
  }

  return cachedCollections;
}
 

private void downloadFromZK(SolrZkClient zkClient, String zkPath, URI dir) throws IOException {
  try {
    if (!repository.exists(dir)) {
      repository.createDirectory(dir);
    }
    List<String> files = zkClient.getChildren(zkPath, null, true);
    for (String file : files) {
      List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);
      if (children.size() == 0) {
        log.debug("Writing file {}", file);
        byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
        try (OutputStream os = repository.createOutput(repository.resolve(dir, file))) {
          os.write(data);
        }
      } else {
        downloadFromZK(zkClient, zkPath + "/" + file, repository.resolve(dir, file));
      }
    }
  } catch (KeeperException | InterruptedException e) {
    throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(),
        SolrZkClient.checkInterrupted(e));
  }
}
 

/**
 * This method returns the {@linkplain CollectionSnapshotMetaData} for each named snapshot for the specified collection in Zookeeper.
 *
 * @param zkClient Zookeeper client
 * @param collectionName The name of the collection
 * @return the {@linkplain CollectionSnapshotMetaData} for each named snapshot
 * @throws InterruptedException In case of thread interruption.
 * @throws KeeperException In case of Zookeeper error
 */
public static Collection<CollectionSnapshotMetaData> listSnapshots(SolrZkClient zkClient, String collectionName)
    throws InterruptedException, KeeperException {
  Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
  String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.empty());

  try {
    Collection<String> snapshots = zkClient.getChildren(zkPath, null, true);
    for (String snapshot : snapshots) {
      Optional<CollectionSnapshotMetaData> s = getCollectionLevelSnapshot(zkClient, collectionName, snapshot);
      if (s.isPresent()) {
        result.add(s.get());
      }
    }
  } catch (KeeperException ex) {
    // Gracefully handle the case when the zk node doesn't exist (e.g. due to a concurrent delete collection operation).
    if ( ex.code() != KeeperException.Code.NONODE ) {
      throw ex;
    }
  }
  return result;
}
 

protected void assertOpenACLUnsafeAllover(SolrZkClient zkClient, String path, List<String> verifiedList) throws Exception {
  List<ACL> acls = zkClient.getSolrZooKeeper().getACL(path, new Stat());
  if (log.isInfoEnabled()) {
    log.info("Verifying {}", path);
  }
  if (ZooDefs.CONFIG_NODE.equals(path)) {
    // Treat this node specially, from the ZK docs:
    // The dynamic configuration is stored in a special znode ZooDefs.CONFIG_NODE = /zookeeper/config.
    // This node by default is read only for all users, except super user and
    // users that's explicitly configured for write access.
    assertEquals("Path " + path + " does not have READ_ACL_UNSAFE", ZooDefs.Ids.READ_ACL_UNSAFE, acls);
  } else {
    assertEquals("Path " + path + " does not have OPEN_ACL_UNSAFE", ZooDefs.Ids.OPEN_ACL_UNSAFE, acls);
  }
  verifiedList.add(path);
  List<String> children = zkClient.getChildren(path, null, false);
  for (String child : children) {
    assertOpenACLUnsafeAllover(zkClient, path + ((path.endsWith("/")) ? "" : "/") + child, verifiedList);
  }
}
 

public static List<String> getSortedOverseerNodeNames(SolrZkClient zk) throws KeeperException, InterruptedException {
  List<String> children = null;
  try {
    children = zk.getChildren(Overseer.OVERSEER_ELECT + LeaderElector.ELECTION_NODE, null, true);
  } catch (Exception e) {
    log.warn("error ", e);
    return new ArrayList<>();
  }
  LeaderElector.sortSeqs(children);
  ArrayList<String> nodeNames = new ArrayList<>(children.size());
  for (String c : children) nodeNames.add(LeaderElector.getNodeName(c));
  return nodeNames;
}
 

public static List<String> getSortedElectionNodes(SolrZkClient zk, String path) throws KeeperException, InterruptedException {
  List<String> children = null;
  try {
    children = zk.getChildren(path, null, true);
    LeaderElector.sortSeqs(children);
    return children;
  } catch (Exception e) {
    throw e;
  }

}
 

private void deleteThemAll(SolrZkClient zkClient, String node) throws KeeperException, InterruptedException {
  List<String> kids = zkClient.getChildren(node, null, true);
  for (String kid : kids) {
    deleteThemAll(zkClient, node + "/" + kid);
  }
  zkClient.delete(node, -1, true);
}
 

/** returns the true set of live nodes (currently in zk) as a sorted list */
private static List<String> getTrueLiveNodesFromZk() throws Exception {
  SolrZkClient client = newSolrZkClient();
  try {
    ArrayList<String> result = new ArrayList<>(client.getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true));
    Collections.sort(result);
    return result;
  } finally {
    client.close();
  }
}
 

protected final void assertFileNotInZooKeeper(SolrZkClient zkClient, String parent, String fileName) throws Exception {
  List<String> kids = zkClient.getChildren(parent, null, true);
  for (String kid : kids) {
    if (kid.equalsIgnoreCase(fileName)) {
      String rawContent = new String(zkClient.getData(fileName, null, null, true), StandardCharsets.UTF_8);
      fail("File '" + fileName + "' was unexpectedly found in ZooKeeper.  Content starts with '"
          + rawContent.substring(0, 100) + " [...]'");
    }
  }
}
 

private void showFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp,
    CoreContainer coreContainer) throws KeeperException,
    InterruptedException, UnsupportedEncodingException {

  SolrZkClient zkClient = coreContainer.getZkController().getZkClient();

  String adminFile = getAdminFileFromZooKeeper(req, rsp, zkClient, hiddenFiles);

  if (adminFile == null) {
    return;
  }

  // Show a directory listing
  List<String> children = zkClient.getChildren(adminFile, null, true);
  if (children.size() > 0) {
    
    NamedList<SimpleOrderedMap<Object>> files = new SimpleOrderedMap<>();
    for (String f : children) {
      if (isHiddenFile(req, rsp, f, false, hiddenFiles)) {
        continue;
      }

      SimpleOrderedMap<Object> fileInfo = new SimpleOrderedMap<>();
      files.add(f, fileInfo);
      List<String> fchildren = zkClient.getChildren(adminFile + "/" + f, null, true);
      if (fchildren.size() > 0) {
        fileInfo.add("directory", true);
      } else {
        // TODO? content type
        fileInfo.add("size", f.length());
      }
      // TODO: ?
      // fileInfo.add( "modified", new Date( f.lastModified() ) );
    }
    rsp.add("files", files);
  } else {
    // Include the file contents
    // The file logic depends on RawResponseWriter, so force its use.
    ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
    params.set(CommonParams.WT, "raw");
    req.setParams(params);
    ContentStreamBase content = new ContentStreamBase.ByteArrayStream(zkClient.getData(adminFile, null, null, true), adminFile);
    content.setContentType(req.getParams().get(USE_CONTENT_TYPE));
    
    rsp.add(RawResponseWriter.CONTENT, content);
  }
  rsp.setHttpCaching(false);
}
 

private List<String> getElectionNodes(String collection, String shard, SolrZkClient client) throws KeeperException, InterruptedException {
  return client.getChildren("/collections/"+collection+"/leader_elect/"+shard+LeaderElector.ELECTION_NODE, null, true);
}
 

@Test
@SuppressWarnings({"try"})
public void testMultipleWatchesAsync() throws Exception {
  try (ZkConnection conn = new ZkConnection()) {
    final SolrZkClient zkClient = conn.getClient();
    zkClient.makePath("/collections", true);

    final int numColls = random().nextInt(100);
    final CountDownLatch latch = new CountDownLatch(numColls);
    final CountDownLatch watchesDone = new CountDownLatch(numColls);
    final Set<String> collectionsInProgress = new HashSet<>(numColls);
    AtomicInteger maxCollectionsInProgress = new AtomicInteger();

    for (int i = 1; i <= numColls; i ++) {
      String collPath = "/collections/collection" + i;
      zkClient.makePath(collPath, true);
      zkClient.getChildren(collPath, new Watcher() {
        @Override
        public void process(WatchedEvent event) {
          synchronized (collectionsInProgress) {
            collectionsInProgress.add(event.getPath()); // Will be something like /collections/collection##
            maxCollectionsInProgress.set(Math.max(maxCollectionsInProgress.get(), collectionsInProgress.size()));
          }
          latch.countDown();
          try {
            latch.await(10000, TimeUnit.MILLISECONDS);
          }
          catch (InterruptedException e) {}
          synchronized (collectionsInProgress) {
            collectionsInProgress.remove(event.getPath());
          }
          watchesDone.countDown();
        }
      }, true);
    }

    for (int i = 1; i <= numColls; i ++) {
      String shardsPath = "/collections/collection" + i + "/shards";
      zkClient.makePath(shardsPath, true);
    }

    assertTrue(latch.await(10000, TimeUnit.MILLISECONDS));
    assertEquals("All collections should have been processed in parallel", numColls, maxCollectionsInProgress.get());
    
    // just as sanity check for the test:
    assertTrue(watchesDone.await(10000, TimeUnit.MILLISECONDS));
    synchronized (collectionsInProgress) {
      assertEquals(0, collectionsInProgress.size());
    }
  }
}
 

@SuppressWarnings({"try"})
public void testWatchChildren() throws Exception {
  try (ZkConnection conn = new ZkConnection ()) {
    final SolrZkClient zkClient = conn.getClient();
    final AtomicInteger cnt = new AtomicInteger();
    final CountDownLatch latch = new CountDownLatch(1);

    zkClient.makePath("/collections", true);

    zkClient.getChildren("/collections", new Watcher() {

      @Override
      public void process(WatchedEvent event) {
        cnt.incrementAndGet();
        // remake watch
        try {
          zkClient.getChildren("/collections", this, true);
          latch.countDown();
        } catch (KeeperException | InterruptedException e) {
          throw new RuntimeException(e);
        }
      }
    }, true);

    zkClient.makePath("/collections/collection99/shards", true);
    latch.await(); //wait until watch has been re-created

    zkClient.makePath("collections/collection99/config=collection1", true);

    zkClient.makePath("collections/collection99/config=collection3", true);
    
    zkClient.makePath("/collections/collection97/shards", true);
    
    // pause for the watches to fire
    Thread.sleep(700);
    
    if (cnt.intValue() < 2) {
      Thread.sleep(4000); // wait a bit more
    }
    
    if (cnt.intValue() < 2) {
      Thread.sleep(4000); // wait a bit more
    }
    
    assertEquals(2, cnt.intValue());

  }
}