Java Code Examples for org.apache.solr.common.cloud.SolrZkClient#getData()

public boolean doIfConfigExists(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, String separator) throws IOException {
  logger.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection());
  try {
    File[] listOfFiles = getConfigSetFolder().listFiles();
    if (listOfFiles == null)
      return false;
    byte[] data = zkClient.getData(String.format("%s/%s/%s", CONFIGS_ZKNODE, solrPropsConfig.getConfigName(), getConfigFileName()), null, null, true);

    for (File file : listOfFiles) {
      if (file.getName().equals(getConfigFileName()) && updateConfigIfNeeded(solrPropsConfig, zkClient, file, separator, data)) {
        return true;
      }
    }
    return false;
  } catch (KeeperException | InterruptedException e) {
    throw new IOException("Error downloading files from zookeeper path " + solrPropsConfig.getConfigName(),
            SolrZkClient.checkInterrupted(e));
  }
}
 

@BeforeClass
public static void setupCluster() throws Exception {
  System.setProperty("enable.runtime.lib", "true");
  configureCluster(3)
      .addConfig("conf", configset("cloud-minimal"))
      .configure();
  SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient();
  String path = ZkStateReader.CONFIGS_ZKNODE + "/conf/solrconfig.xml";
  byte[] data = zkClient.getData(path, null, null, true);

  String solrconfigStr = new String(data, StandardCharsets.UTF_8);
  zkClient.setData(path, solrconfigStr.replace("</config>",
      "<updateRequestProcessorChain name=\"test_urp\" processor=\"testURP\" default=\"true\">\n" +
      "    <processor class=\"solr.RunUpdateProcessorFactory\"/>\n" +
      "  </updateRequestProcessorChain>\n" +
      "\n" +
      "  <updateProcessor class=\"runtimecode.TestURP\" name=\"testURP\" runtimeLib=\"true\"></updateProcessor>\n" +
      "</config>").getBytes(StandardCharsets.UTF_8), true );


  CollectionAdminRequest.createCollection(COLLECTION, "conf", 3, 1).process(cluster.getSolrClient());
  waitForState("", COLLECTION, clusterShape(3, 3));
}
 

private void downloadFromZK(SolrZkClient zkClient, String zkPath, URI dir) throws IOException {
  try {
    if (!repository.exists(dir)) {
      repository.createDirectory(dir);
    }
    List<String> files = zkClient.getChildren(zkPath, null, true);
    for (String file : files) {
      List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);
      if (children.size() == 0) {
        log.debug("Writing file {}", file);
        byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
        try (OutputStream os = repository.createOutput(repository.resolve(dir, file))) {
          os.write(data);
        }
      } else {
        downloadFromZK(zkClient, zkPath + "/" + file, repository.resolve(dir, file));
      }
    }
  } catch (KeeperException | InterruptedException e) {
    throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(),
        SolrZkClient.checkInterrupted(e));
  }
}
 

private void uploadConfigSetWithAssertions(String configSetName, String suffix, String username, String password) throws Exception {
  SolrZkClient zkClient = new SolrZkClient(solrCluster.getZkServer().getZkAddress(),
      AbstractZkTestCase.TIMEOUT, 45000, null);
  try {
    long statusCode = uploadConfigSet(configSetName, suffix, username, password, zkClient);
    assertEquals(0l, statusCode);

    assertTrue("managed-schema file should have been uploaded",
        zkClient.exists("/configs/"+configSetName+suffix+"/managed-schema", true));
    assertTrue("managed-schema file contents on zookeeper are not exactly same as that of the file uploaded in config",
        Arrays.equals(zkClient.getData("/configs/"+configSetName+suffix+"/managed-schema", null, null, true),
            readFile("solr/configsets/upload/"+configSetName+"/managed-schema")));

    assertTrue("solrconfig.xml file should have been uploaded",
        zkClient.exists("/configs/"+configSetName+suffix+"/solrconfig.xml", true));
    byte data[] = zkClient.getData("/configs/"+configSetName+suffix, null, null, true);
    //assertEquals("{\"trusted\": false}", new String(data, StandardCharsets.UTF_8));
    assertTrue("solrconfig.xml file contents on zookeeper are not exactly same as that of the file uploaded in config",
        Arrays.equals(zkClient.getData("/configs/"+configSetName+suffix+"/solrconfig.xml", null, null, true),
            readFile("solr/configsets/upload/"+configSetName+"/solrconfig.xml")));
  } finally {
    zkClient.close();
  }
}
 

@SuppressWarnings({"rawtypes"})
private NamedList getConfigSetPropertiesFromZk(
    SolrZkClient zkClient, String path) throws Exception {
  byte [] oldPropsData = null;
  try {
    oldPropsData = zkClient.getData(path, null, null, true);
  } catch (KeeperException.NoNodeException e) {
    // okay, properties just don't exist
  }

  if (oldPropsData != null) {
    InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);
    try {
      return ConfigSetProperties.readFromInputStream(reader);
    } finally {
      reader.close();
    }
  }
  return null;
}
 

/**
 * Returns config value given collection name
 * Borrowed heavily from Solr's ZKController.
 */
public String readConfigName(SolrZkClient zkClient, String collection)
throws KeeperException, InterruptedException {
  if (collection == null) {
    throw new IllegalArgumentException("collection must not be null");
  }
  String configName = null;

  // first check for alias
  byte[] aliasData = zkClient.getData(ZkStateReader.ALIASES, null, null, true);
  Aliases aliases = ClusterState.load(aliasData);
  String alias = aliases.getCollectionAlias(collection);
  if (alias != null) {
    List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
    if (aliasList.size() > 1) {
      throw new IllegalArgumentException("collection cannot be an alias that maps to multiple collections");
    }
    collection = aliasList.get(0);
  }
  
  String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
  if (LOG.isInfoEnabled()) {
    LOG.info("Load collection config from:" + path);
  }
  byte[] data = zkClient.getData(path, null, null, true);
  
  if(data != null) {
    ZkNodeProps props = ZkNodeProps.load(data);
    configName = props.getStr(ZkController.CONFIGNAME_PROP);
  }
  
  if (configName != null && !zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true)) {
    LOG.error("Specified config does not exist in ZooKeeper:" + configName);
    throw new IllegalArgumentException("Specified config does not exist in ZooKeeper:"
      + configName);
  }

  return configName;
}
 

private String checkForAlias(SolrZkClient zkClient, String collection) throws KeeperException,
    InterruptedException {
  byte[] aliasData = zkClient.getData(ZkStateReader.ALIASES, null, null, true);
  Aliases aliases = ClusterState.load(aliasData);
  String alias = aliases.getCollectionAlias(collection);
  if (alias != null) {
    List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
    if (aliasList.size() > 1) {
      throw new IllegalArgumentException(
          "collection cannot be an alias that maps to multiple collections");
    }
    collection = aliasList.get(0);
  }
  return collection;
}
 

/**
 * Returns config value given collection name Borrowed heavily from Solr's ZKController.
 */
public String readConfigName(SolrZkClient zkClient, String collection) throws KeeperException,
    InterruptedException {
  if (collection == null) {
    throw new IllegalArgumentException("collection must not be null");
  }
  String configName = null;

  // first check for alias
  collection = checkForAlias(zkClient, collection);

  String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
  if (LOG.isInfoEnabled()) {
    LOG.info("Load collection config from:" + path);
  }
  byte[] data = zkClient.getData(path, null, null, true);

  if (data != null) {
    ZkNodeProps props = ZkNodeProps.load(data);
    configName = props.getStr(ZkController.CONFIGNAME_PROP);
  }

  if (configName != null
      && !zkClient.exists(ZkController.CONFIGS_ZKNODE + "/" + configName, true)) {
    LOG.error("Specified config does not exist in ZooKeeper:" + configName);
    throw new IllegalArgumentException("Specified config does not exist in ZooKeeper:"
        + configName);
  }

  return configName;
}
 

protected final void assertFileNotInZooKeeper(SolrZkClient zkClient, String parent, String fileName) throws Exception {
  List<String> kids = zkClient.getChildren(parent, null, true);
  for (String kid : kids) {
    if (kid.equalsIgnoreCase(fileName)) {
      String rawContent = new String(zkClient.getData(fileName, null, null, true), StandardCharsets.UTF_8);
      fail("File '" + fileName + "' was unexpectedly found in ZooKeeper.  Content starts with '"
          + rawContent.substring(0, 100) + " [...]'");
    }
  }
}
 

private void checkConfReload(SolrZkClient client, String resPath, String name, String uri) throws Exception {
  Stat stat =  new Stat();
  byte[] data = null;
  try {
    data = client.getData(resPath, null, stat, true);
  } catch (KeeperException.NoNodeException e) {
    data = "{}".getBytes(StandardCharsets.UTF_8);
    log.info("creating_node {}",resPath);
    client.create(resPath,data, CreateMode.PERSISTENT,true);
  }
  long startTime = System.nanoTime();
  Stat newStat = client.setData(resPath, data, true);
  client.setData("/configs/conf1", new byte[]{1}, true);
  assertTrue(newStat.getVersion() > stat.getVersion());
  if (log.isInfoEnabled()) {
    log.info("new_version {}", newStat.getVersion());
  }
  Integer newVersion = newStat.getVersion();
  long maxTimeoutSeconds = 60;
  DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection("collection1");
  List<String> urls = new ArrayList<>();
  for (Slice slice : coll.getSlices()) {
    for (Replica replica : slice.getReplicas())
      urls.add(""+replica.get(ZkStateReader.BASE_URL_PROP) + "/"+replica.get(ZkStateReader.CORE_NAME_PROP));
  }
  HashSet<String> succeeded = new HashSet<>();

  while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds){
    Thread.sleep(50);
    for (String url : urls) {
      MapWriter respMap = getAsMap(url + uri);
      if (String.valueOf(newVersion).equals(respMap._getStr(asList(name, "znodeVersion"), null))) {
        succeeded.add(url);
      }
    }
    if(succeeded.size() == urls.size()) break;
    succeeded.clear();
  }
  assertEquals(StrUtils.formatString("tried these servers {0} succeeded only in {1} ", urls, succeeded) , urls.size(), succeeded.size());
}
 

/**
 * <p>Verifies if /clusterstate.json exists in Zookeepeer, and if it does and is not empty, refuses to start and outputs
 * a helpful message regarding collection migration.</p>
 *
 * <p>If /clusterstate.json exists and is empty, it is removed.</p>
 */
private void checkNoOldClusterstate(final SolrZkClient zkClient) throws InterruptedException {
  try {
    if (!zkClient.exists(ZkStateReader.UNSUPPORTED_CLUSTER_STATE, true)) {
      return;
    }

    final byte[] data = zkClient.getData(ZkStateReader.UNSUPPORTED_CLUSTER_STATE, null, null, true);

    if (Arrays.equals("{}".getBytes(StandardCharsets.UTF_8), data)) {
      // Empty json. This log will only occur once.
      log.warn("{} no longer supported starting with Solr 9. Found empty file on Zookeeper, deleting it.", ZkStateReader.UNSUPPORTED_CLUSTER_STATE);
      zkClient.delete(ZkStateReader.UNSUPPORTED_CLUSTER_STATE, -1, true);
    } else {
      // /clusterstate.json not empty: refuse to start but do not automatically delete. A bit of a pain but user shouldn't
      // have older collections at this stage anyway.
      String message = ZkStateReader.UNSUPPORTED_CLUSTER_STATE + " no longer supported starting with Solr 9. "
          + "It is present and not empty. Cannot start Solr. Please first migrate collections to stateFormat=2 using an "
          + "older version of Solr or if you don't care about the data then delete the file from "
          + "Zookeeper using a command line tool, for example: bin/solr zk rm /clusterstate.json -z host:port";
      log.error(message);
      throw new SolrException(SolrException.ErrorCode.INVALID_STATE, message);
    }
  } catch (KeeperException e) {
    // Convert checked exception to one acceptable by the caller (see also init() further down)
    log.error("", e);
    throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
  }
}
 

public static String getLeaderId(SolrZkClient zkClient) throws KeeperException,InterruptedException{
  byte[] data = null;
  try {
    data = zkClient.getData(Overseer.OVERSEER_ELECT + "/leader", null, new Stat(), true);
  } catch (KeeperException.NoNodeException e) {
    return null;
  }
  @SuppressWarnings({"rawtypes"})
  Map m = (Map) Utils.fromJSON(data);
  return  (String) m.get(ID);
}
 

/**
 * Assumes data in ZooKeeper is a JSON string, deserializes it and returns as a Map
 *
 * @param zkClient        the zookeeper client
 * @param path            the path to the znode being read
 * @param retryOnConnLoss whether to retry the operation automatically on connection loss, see {@link org.apache.solr.common.cloud.ZkCmdExecutor#retryOperation(ZkOperation)}
 * @return a Map if the node exists and contains valid JSON or an empty map if znode does not exist or has a null data
 */
@SuppressWarnings({"unchecked"})
public static Map<String, Object> getJson(SolrZkClient zkClient, String path, boolean retryOnConnLoss) throws KeeperException, InterruptedException {
  try {
    byte[] bytes = zkClient.getData(path, null, null, retryOnConnLoss);
    if (bytes != null && bytes.length > 0) {
      return (Map<String, Object>) Utils.fromJSON(bytes);
    }
  } catch (KeeperException.NoNodeException e) {
    return Collections.emptyMap();
  }
  return Collections.emptyMap();
}
 

@SuppressWarnings({"try"})
public void testSkipPathPartsOnMakePath() throws Exception {
  try (ZkConnection conn = new ZkConnection()) {
    final SolrZkClient zkClient = conn.getClient();

    zkClient.makePath("/test", true);

    // should work
    zkClient.makePath("/test/path/here", (byte[]) null, CreateMode.PERSISTENT, (Watcher) null, true, true, 1);

    zkClient.clean("/");

    // should not work
    KeeperException e =expectThrows(KeeperException.NoNodeException.class,
        "We should not be able to create this path",
        () -> zkClient.makePath("/test/path/here", (byte[]) null, CreateMode.PERSISTENT, (Watcher) null, true, true, 1));

    zkClient.clean("/");

    ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(30000);
    expectThrows(KeeperException.NoNodeException.class,
        "We should not be able to create this path",
        () -> zkCmdExecutor.ensureExists("/collection/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2));

    zkClient.makePath("/collection", true);

    expectThrows(KeeperException.NoNodeException.class,
        "We should not be able to create this path",
        () -> zkCmdExecutor.ensureExists("/collections/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2));
    zkClient.makePath("/collection/collection", true);
 
    byte[] bytes = new byte[10];
    zkCmdExecutor.ensureExists("/collection/collection", bytes, CreateMode.PERSISTENT, zkClient, 2);
    
    byte[] returnedBytes = zkClient.getData("/collection/collection", null, null, true);
    
    assertNull("We skipped 2 path parts, so data won't be written", returnedBytes);

    zkClient.makePath("/collection/collection/leader", true);

    zkCmdExecutor.ensureExists("/collection/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2);

  }
}
 

private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException {
  if (zkClient.exists(PackageUtils.REPOSITORIES_ZK_PATH, true)) {
    return new String(zkClient.getData(PackageUtils.REPOSITORIES_ZK_PATH, null, null, true), "UTF-8");
  }
  return "[]";
}
 

private void showFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp,
    CoreContainer coreContainer) throws KeeperException,
    InterruptedException, UnsupportedEncodingException {

  SolrZkClient zkClient = coreContainer.getZkController().getZkClient();

  String adminFile = getAdminFileFromZooKeeper(req, rsp, zkClient, hiddenFiles);

  if (adminFile == null) {
    return;
  }

  // Show a directory listing
  List<String> children = zkClient.getChildren(adminFile, null, true);
  if (children.size() > 0) {
    
    NamedList<SimpleOrderedMap<Object>> files = new SimpleOrderedMap<>();
    for (String f : children) {
      if (isHiddenFile(req, rsp, f, false, hiddenFiles)) {
        continue;
      }

      SimpleOrderedMap<Object> fileInfo = new SimpleOrderedMap<>();
      files.add(f, fileInfo);
      List<String> fchildren = zkClient.getChildren(adminFile + "/" + f, null, true);
      if (fchildren.size() > 0) {
        fileInfo.add("directory", true);
      } else {
        // TODO? content type
        fileInfo.add("size", f.length());
      }
      // TODO: ?
      // fileInfo.add( "modified", new Date( f.lastModified() ) );
    }
    rsp.add("files", files);
  } else {
    // Include the file contents
    // The file logic depends on RawResponseWriter, so force its use.
    ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
    params.set(CommonParams.WT, "raw");
    req.setParams(params);
    ContentStreamBase content = new ContentStreamBase.ByteArrayStream(zkClient.getData(adminFile, null, null, true), adminFile);
    content.setContentType(req.getParams().get(USE_CONTENT_TYPE));
    
    rsp.add(RawResponseWriter.CONTENT, content);
  }
  rsp.setHttpCaching(false);
}
 

private String getFileContentFromZooKeeper(SolrZkClient zkClient, String fileName)
    throws IOException, SolrServerException, KeeperException, InterruptedException {

  return (new String(zkClient.getData(fileName, null, null, true), StandardCharsets.UTF_8));

}