Java Code Examples for org.apache.solr.common.cloud.ZkNodeProps#load()

private String getLeaderUrl(final String collection, final String slice)
    throws KeeperException, InterruptedException {
  int iterCount = 60;
  while (iterCount-- > 0) {
    try {
      byte[] data = zkClient.getData(
          ZkStateReader.getShardLeadersPath(collection, slice), null, null,
          true);
      ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(
          ZkNodeProps.load(data));
      return leaderProps.getCoreUrl();
    } catch (NoNodeException | SessionExpiredException e) {
      Thread.sleep(500);
    }
  }
  zkClient.printLayoutToStream(System.out);
  throw new RuntimeException("Could not get leader props for " + collection + " " + slice);
}
 

@Test
public void testBasic() throws IOException {
  
  Map<String,Object> props = new HashMap<>();
  props.put("prop1", "value1");
  props.put("prop2", "value2");
  props.put("prop3", "value3");
  props.put("prop4", "value4");
  props.put("prop5", "value5");
  props.put("prop6", "value6");
  
  ZkNodeProps zkProps = new ZkNodeProps(props);
  byte[] bytes = Utils.toJSON(zkProps);
  ZkNodeProps props2 = ZkNodeProps.load(bytes);

  props.forEach((s, o) -> assertEquals(o, props2.get(s)));
  SimplePostTool.BAOS baos = new SimplePostTool.BAOS();
  try (JavaBinCodec jbc = new JavaBinCodec()) {
    jbc.marshal(zkProps.getProperties(), baos);
  }
  bytes = baos.toByteArray();
  System.out.println("BIN size : " + bytes.length);
  ZkNodeProps props3 = ZkNodeProps.load(bytes);
  props.forEach((s, o) -> assertEquals(o, props3.get(s)));
}
 

private static void testAutoAddReplicas() throws Exception {
  TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  while (!timeOut.hasTimedOut()) {
    byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
    ZkNodeProps loaded = ZkNodeProps.load(data);
    @SuppressWarnings({"rawtypes"})
    Map triggers = (Map) loaded.get("triggers");
    if (triggers != null && triggers.containsKey(".auto_add_replicas")) {
      @SuppressWarnings({"unchecked"})
      Map<String, Object> autoAddReplicasTrigger = (Map<String, Object>) triggers.get(".auto_add_replicas");
      assertNotNull(autoAddReplicasTrigger);
      @SuppressWarnings({"unchecked"})
      List<Map<String, Object>> actions = (List<Map<String, Object>>) autoAddReplicasTrigger.get("actions");
      assertNotNull(actions);
      assertEquals(2, actions.size());
      assertEquals("auto_add_replicas_plan", actions.get(0).get("name").toString());
      assertEquals("solr.AutoAddReplicasPlanAction", actions.get(0).get("class").toString());
      break;
    } else {
      Thread.sleep(300);
    }
  }
  if (timeOut.hasTimedOut()) {
    fail("Timeout waiting for .auto_add_replicas being created");
  }
}
 

@Override
public void distribSetUp() throws Exception {
  super.distribSetUp();
  // ignoreException(".*");
  
  cloudInit = false;
  
  if (sliceCount > 0) {
    System.setProperty("numShards", Integer.toString(sliceCount));
  } else {
    System.clearProperty("numShards");
  }

  if (isSSLMode()) {
    System.clearProperty("urlScheme");
    try (ZkStateReader zkStateReader = new ZkStateReader(zkServer.getZkAddress(),
        AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT)) {
      try {
        zkStateReader.getZkClient().create(ZkStateReader.CLUSTER_PROPS,
            Utils.toJSON(Collections.singletonMap("urlScheme", "https")),
            CreateMode.PERSISTENT, true);
      } catch (KeeperException.NodeExistsException e) {
        ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData(ZkStateReader.CLUSTER_PROPS,
            null, null, true));
        zkStateReader.getZkClient().setData(ZkStateReader.CLUSTER_PROPS, Utils.toJSON(props.plus("urlScheme", "https")), true);
      }
    }
  }
  if (useTlogReplicas()) {
    log.info("Will use {} replicas unless explicitly asked otherwise", Replica.Type.TLOG);
  } else {
    log.info("Will use {} replicas unless explicitly asked otherwise", Replica.Type.NRT);
  }
}
 

private void checkIfIAmLeader() throws KeeperException, InterruptedException {
  SolrZkClient zkClient = core.getCoreContainer().getZkController().getZkClient();
  ZkNodeProps props = ZkNodeProps.load(zkClient.getData(CdcrLeaderStateManager.this.getZnodePath(), null, null, true));
  if (props != null) {
    CdcrLeaderStateManager.this.setAmILeader(props.get("core").equals(core.getName()));
  }
}
 

/**
 * Returns true if the queue contains a task with the specified async id.
 */
public boolean containsTaskWithRequestId(String requestIdKey, String requestId)
    throws KeeperException, InterruptedException {

  List<String> childNames = zookeeper.getChildren(dir, null, true);
  stats.setQueueLength(childNames.size());
  for (String childName : childNames) {
    if (childName != null && childName.startsWith(PREFIX)) {
      try {
        byte[] data = zookeeper.getData(dir + "/" + childName, null, null, true);
        if (data != null) {
          ZkNodeProps message = ZkNodeProps.load(data);
          if (message.containsKey(requestIdKey)) {
            if (log.isDebugEnabled()) {
              log.debug("Looking for {}, found {}", message.get(requestIdKey), requestId);
            }
            if(message.get(requestIdKey).equals(requestId)) return true;
          }
        }
      } catch (KeeperException.NoNodeException e) {
        // Another client removed the node first, try next
      }
    }
  }

  return false;
}
 

@Test
public void testSpecificConfigsets() throws Exception {
  CollectionAdminRequest.createCollection("withconfigset2", "conf2", 1, 1).process(cluster.getSolrClient());
  byte[] data = zkClient().getData(ZkStateReader.COLLECTIONS_ZKNODE + "/" + "withconfigset2", null, null, true);
  assertNotNull(data);
  ZkNodeProps props = ZkNodeProps.load(data);
  String configName = props.getStr(ZkController.CONFIGNAME_PROP);
  assertEquals("conf2", configName);
}
 

@Override
public void distribSetUp() throws Exception {
  super.distribSetUp();

  if (shardCount > 0) {
    System.setProperty("numShards", Integer.toString(shardCount));
  } else {
    System.clearProperty("numShards");
  }

  if (isSSLMode()) {
    System.clearProperty("urlScheme");
    ZkStateReader zkStateReader = new ZkStateReader(zkServer.getZkAddress(),
        AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT);
    try {
      zkStateReader.getZkClient().create(ZkStateReader.CLUSTER_PROPS,
          Utils.toJSON(Collections.singletonMap("urlScheme", "https")),
          CreateMode.PERSISTENT, true);
    } catch (KeeperException.NodeExistsException e) {
      ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData(ZkStateReader.CLUSTER_PROPS,
          null, null, true));
      props = props.plus("urlScheme", "https");
      zkStateReader.getZkClient().setData(CLUSTER_PROPS, Utils.toJSON(props), true);
    } finally {
      zkStateReader.close();
    }
  }
}
 

private String showQpeek(ZkDistributedQueue q) throws KeeperException, InterruptedException {
  if (q == null) {
    return "";
  }
  byte[] bytes = q.peek();
  if (bytes == null) {
    return "";
  }

  ZkNodeProps json = ZkNodeProps.load(bytes);
  return json.toString();
}
 

private void handleCreateCollMessage(byte[] bytes) {
  log.info("track created replicas / collections");
  try {
    ZkNodeProps props = ZkNodeProps.load(bytes);
    if (CollectionParams.CollectionAction.CREATE.isEqual(props.getStr("operation"))) {
      String collName = props.getStr("name");
      if (collName != null) collectionsSet.put(collName, new ClusterState.CollectionRef(
          new DocCollection(collName, new HashMap<>(), props.getProperties(), DocRouter.DEFAULT)));
    }
    if (CollectionParams.CollectionAction.ADDREPLICA.isEqual(props.getStr("operation"))) {
      replicas.add(props);
    }
  } catch (Exception e) {}
}
 

/**
 * Returns config value given collection name Borrowed heavily from Solr's ZKController.
 */
public String readConfigName(SolrZkClient zkClient, String collection) throws KeeperException,
    InterruptedException {
  if (collection == null) {
    throw new IllegalArgumentException("collection must not be null");
  }
  String configName = null;

  // first check for alias
  collection = checkForAlias(zkClient, collection);

  String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
  if (LOG.isInfoEnabled()) {
    LOG.info("Load collection config from:" + path);
  }
  byte[] data = zkClient.getData(path, null, null, true);

  if (data != null) {
    ZkNodeProps props = ZkNodeProps.load(data);
    configName = props.getStr(ZkController.CONFIGNAME_PROP);
  }

  if (configName != null
      && !zkClient.exists(ZkController.CONFIGS_ZKNODE + "/" + configName, true)) {
    LOG.error("Specified config does not exist in ZooKeeper:" + configName);
    throw new IllegalArgumentException("Specified config does not exist in ZooKeeper:"
        + configName);
  }

  return configName;
}
 

/**
 * Returns config value given collection name
 * Borrowed heavily from Solr's ZKController.
 */
public String readConfigName(SolrZkClient zkClient, String collection)
throws KeeperException, InterruptedException {
  if (collection == null) {
    throw new IllegalArgumentException("collection must not be null");
  }
  String configName = null;

  // first check for alias
  byte[] aliasData = zkClient.getData(ZkStateReader.ALIASES, null, null, true);
  Aliases aliases = ClusterState.load(aliasData);
  String alias = aliases.getCollectionAlias(collection);
  if (alias != null) {
    List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
    if (aliasList.size() > 1) {
      throw new IllegalArgumentException("collection cannot be an alias that maps to multiple collections");
    }
    collection = aliasList.get(0);
  }
  
  String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
  if (LOG.isInfoEnabled()) {
    LOG.info("Load collection config from:" + path);
  }
  byte[] data = zkClient.getData(path, null, null, true);
  
  if(data != null) {
    ZkNodeProps props = ZkNodeProps.load(data);
    configName = props.getStr(ZkController.CONFIGNAME_PROP);
  }
  
  if (configName != null && !zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true)) {
    LOG.error("Specified config does not exist in ZooKeeper:" + configName);
    throw new IllegalArgumentException("Specified config does not exist in ZooKeeper:"
      + configName);
  }

  return configName;
}
 

private static void getConfName(DistribStateManager stateManager, String collection, String collectionPath, Map<String,Object> collectionProps) throws IOException,
    KeeperException, InterruptedException {
  // check for configName
  log.debug("Looking for collection configName");
  if (collectionProps.containsKey("configName")) {
    if (log.isInfoEnabled()) {
      log.info("configName was passed as a param {}", collectionProps.get("configName"));
    }
    return;
  }

  List<String> configNames = null;
  int retry = 1;
  int retryLimt = 6;
  for (; retry < retryLimt; retry++) {
    if (stateManager.hasData(collectionPath)) {
      VersionedData data = stateManager.getData(collectionPath);
      ZkNodeProps cProps = ZkNodeProps.load(data.getData());
      if (cProps.containsKey(ZkController.CONFIGNAME_PROP)) {
        break;
      }
    }

    try {
      configNames = stateManager.listData(ZkConfigManager.CONFIGS_ZKNODE);
    } catch (NoSuchElementException | NoNodeException e) {
      // just keep trying
    }

    // check if there's a config set with the same name as the collection
    if (configNames != null && configNames.contains(collection)) {
      log.info("Could not find explicit collection configName, but found config name matching collection name - using that set.");
      collectionProps.put(ZkController.CONFIGNAME_PROP, collection);
      break;
    }
    // if _default exists, use that
    if (configNames != null && configNames.contains(ConfigSetsHandlerApi.DEFAULT_CONFIGSET_NAME)) {
      log.info("Could not find explicit collection configName, but found _default config set - using that set.");
      collectionProps.put(ZkController.CONFIGNAME_PROP, ConfigSetsHandlerApi.DEFAULT_CONFIGSET_NAME);
      break;
    }
    // if there is only one conf, use that
    if (configNames != null && configNames.size() == 1) {
      // no config set named, but there is only 1 - use it
      if (log.isInfoEnabled()) {
        log.info("Only one config set found in zk - using it: {}", configNames.get(0));
      }
      collectionProps.put(ZkController.CONFIGNAME_PROP, configNames.get(0));
      break;
    }

    log.info("Could not find collection configName - pausing for 3 seconds and trying again - try: {}", retry);
    Thread.sleep(3000);
  }
  if (retry == retryLimt) {
    log.error("Could not find configName for collection {}", collection);
    throw new ZooKeeperException(
        SolrException.ErrorCode.SERVER_ERROR,
        "Could not find configName for collection " + collection + " found:" + configNames);
  }
}
 

@Test
public void testUpConfigLinkConfigClearZk() throws Exception {
  File tmpDir = createTempDir().toFile();

  // test upconfig
  String confsetname = "confsetone";
  final String[] upconfigArgs;
  if (random().nextBoolean()) {
    upconfigArgs = new String[] {
        "-zkhost", zkServer.getZkAddress(),
        "-cmd", ZkCLI.UPCONFIG,
        "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET,
        "-confname", confsetname};
  } else {
    final String excluderegexOption = (random().nextBoolean() ? "--"+ZkCLI.EXCLUDE_REGEX : "-"+ZkCLI.EXCLUDE_REGEX_SHORT);
    upconfigArgs = new String[] {
        "-zkhost", zkServer.getZkAddress(),
        "-cmd", ZkCLI.UPCONFIG,
        excluderegexOption, ZkCLI.EXCLUDE_REGEX_DEFAULT,
        "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET,
        "-confname", confsetname};
  }
  ZkCLI.main(upconfigArgs);

  assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true));

  // print help
  // ZkCLI.main(new String[0]);

  // test linkconfig
  String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
      "linkconfig", "-collection", "collection1", "-confname", confsetname};
  ZkCLI.main(args);

  ZkNodeProps collectionProps = ZkNodeProps.load(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/collection1", null, null, true));
  assertTrue(collectionProps.containsKey("configName"));
  assertEquals(confsetname, collectionProps.getStr("configName"));

  // test down config
  File confDir = new File(tmpDir,
      "solrtest-confdropspot-" + this.getClass().getName() + "-" + System.nanoTime());
  assertFalse(confDir.exists());

  args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
      "downconfig", "-confdir", confDir.getAbsolutePath(), "-confname", confsetname};
  ZkCLI.main(args);

  File[] files = confDir.listFiles();
  List<String> zkFiles = zkClient.getChildren(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, null, true);
  assertEquals(files.length, zkFiles.size());

  File sourceConfDir = new File(ExternalPaths.TECHPRODUCTS_CONFIGSET);
  // filter out all directories starting with . (e.g. .svn)
  Collection<File> sourceFiles = FileUtils.listFiles(sourceConfDir, TrueFileFilter.INSTANCE, new RegexFileFilter("[^\\.].*"));
  for (File sourceFile :sourceFiles){
      int indexOfRelativePath = sourceFile.getAbsolutePath().lastIndexOf("sample_techproducts_configs" + File.separator + "conf");
      String relativePathofFile = sourceFile.getAbsolutePath().substring(indexOfRelativePath + 33, sourceFile.getAbsolutePath().length());
      File downloadedFile = new File(confDir,relativePathofFile);
      if (ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) {
        assertFalse(sourceFile.getAbsolutePath() + " exists in ZK, downloaded:" + downloadedFile.getAbsolutePath(), downloadedFile.exists());
      } else {
        assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists());
        assertTrue(relativePathofFile+" content changed",FileUtils.contentEquals(sourceFile,downloadedFile));
      }
  }


  // test reset zk
  args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
      "clear", "/"};
  ZkCLI.main(args);

  assertEquals(0, zkClient.getChildren("/", null, true).size());
}