Java Code Examples for org.apache.solr.common.cloud.Slice#getReplicas()

public List<List<String>> extractShardUrls(String zkHost, String collection) {

    DocCollection docCollection = extractDocCollection(zkHost, collection);
    List<Slice> slices = getSortedSlices(docCollection.getSlices());
    List<List<String>> solrUrls = new ArrayList<List<String>>(slices.size());
    for (Slice slice : slices) {
      if (slice.getLeader() == null) {
        throw new IllegalArgumentException("Cannot find SolrCloud slice leader. "
            + "It looks like not all of your shards are registered in ZooKeeper yet");
      }
      Collection<Replica> replicas = slice.getReplicas();
      List<String> urls = new ArrayList<String>(replicas.size());
      for (Replica replica : replicas) {
        ZkCoreNodeProps props = new ZkCoreNodeProps(replica);
        urls.add(props.getCoreUrl());
      }
      solrUrls.add(urls);
    }
    return solrUrls;
  }
 

protected void checkSubShardConsistency(String shard) throws SolrServerException, IOException {
  SolrQuery query = new SolrQuery("*:*").setRows(1000).setFields("id", "_version_");
  query.set("distrib", false);

  ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
  Slice slice = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getSlice(shard);
  long[] numFound = new long[slice.getReplicasMap().size()];
  int c = 0;
  for (Replica replica : slice.getReplicas()) {
    String coreUrl = new ZkCoreNodeProps(replica).getCoreUrl();
    QueryResponse response;
    try (HttpSolrClient client = getHttpSolrClient(coreUrl)) {
      response = client.query(query);
    }
    numFound[c++] = response.getResults().getNumFound();
    if (log.isInfoEnabled()) {
      log.info("Shard: {} Replica: {} has {} docs", shard, coreUrl, String.valueOf(response.getResults().getNumFound()));
    }
    assertTrue("Shard: " + shard + " Replica: " + coreUrl + " has 0 docs", response.getResults().getNumFound() > 0);
  }
  for (int i = 0; i < slice.getReplicasMap().size(); i++) {
    assertEquals(shard + " is not consistent", numFound[0], numFound[i]);
  }
}
 

public static String getAssignedCoreNodeName(DocCollection collection, String forNodeName, String forCoreName) {
  Collection<Slice> slices = collection != null ? collection.getSlices() : null;
  if (slices != null) {
    for (Slice slice : slices) {
      for (Replica replica : slice.getReplicas()) {
        String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
        String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);

        if (nodeName.equals(forNodeName) && core.equals(forCoreName)) {
          return replica.getName();
        }
      }
    }
  }
  return null;
}
 

protected void logReplicaTypesReplicationInfo(String collectionName, ZkStateReader zkStateReader) throws KeeperException, InterruptedException, IOException {
  log.info("## Collecting extra Replica.Type information of the cluster");
  zkStateReader.updateLiveNodes();
  StringBuilder builder = new StringBuilder();
  zkStateReader.forceUpdateCollection(collectionName);
  DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
  for(Slice s:collection.getSlices()) {
    Replica leader = s.getLeader();
    for (Replica r:s.getReplicas()) {
      if (!r.isActive(zkStateReader.getClusterState().getLiveNodes())) {
        builder.append(String.format(Locale.ROOT, "Replica %s not in liveNodes or is not active%s", r.getName(), System.lineSeparator()));
        continue;
      }
      if (r.equals(leader)) {
        builder.append(String.format(Locale.ROOT, "Replica %s is leader%s", r.getName(), System.lineSeparator()));
      }
      logReplicationDetails(r, builder);
    }
  }
  log.info("Summary of the cluster: {}", builder);
}
 

private void verifyPropCorrectlyDistributed(String prop) throws KeeperException, InterruptedException {

    TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);

    String propLC = prop.toLowerCase(Locale.ROOT);
    DocCollection docCollection = null;
    while (timeout.hasTimedOut() == false) {
      forceUpdateCollectionStatus();
      docCollection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);
      int maxPropCount = Integer.MAX_VALUE;
      int minPropCount = Integer.MIN_VALUE;
      for (Slice slice : docCollection.getSlices()) {
        int repCount = 0;
        for (Replica rep : slice.getReplicas()) {
          if (rep.getBool("property." + propLC, false)) {
            repCount++;
          }
        }
        maxPropCount = Math.max(maxPropCount, repCount);
        minPropCount = Math.min(minPropCount, repCount);
      }
      if (Math.abs(maxPropCount - minPropCount) < 2) return;
    }
    log.error("Property {} is not distributed evenly. {}", prop, docCollection);
    fail("Property is not distributed evenly " + prop);
  }
 

private boolean checkdUniquePropPerShard(Map<String, String> uniques, String prop) throws KeeperException, InterruptedException {
  forceUpdateCollectionStatus();
  DocCollection docCollection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);

  for (Slice slice : docCollection.getSlices()) {
    int propfCount = 0;
    for (Replica rep : slice.getReplicas()) {
      if (rep.getBool("property." + prop.toLowerCase(Locale.ROOT), false)) {
        propfCount++;
        uniques.put(slice.getName(), rep.getName());
      }
    }
    if (1 != propfCount) {
      return false;
    }
  }
  return true;
}
 

public static CreateReplica assignReplicaDetails(SolrCloudManager cloudManager, ClusterState clusterState,
                                               ZkNodeProps message, ReplicaPosition replicaPosition) {
  boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);

  String collection = message.getStr(COLLECTION_PROP);
  String node = replicaPosition.node;
  String shard = message.getStr(SHARD_ID_PROP);
  String coreName = message.getStr(CoreAdminParams.NAME);
  String coreNodeName = message.getStr(CoreAdminParams.CORE_NODE_NAME);
  Replica.Type replicaType = replicaPosition.type;

  if (StringUtils.isBlank(coreName)) {
    coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
  }

  log.info("Node Identified {} for creating new replica of shard {} for collection {}", node, shard, collection);
  if (!clusterState.liveNodesContain(node)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
  }
  DocCollection coll = clusterState.getCollection(collection);
  if (coreName == null) {
    coreName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), coll, shard, replicaType);
  } else if (!skipCreateReplicaInClusterState) {
    //Validate that the core name is unique in that collection
    for (Slice slice : coll.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        String replicaCoreName = replica.getStr(CORE_NAME_PROP);
        if (coreName.equals(replicaCoreName)) {
          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" +
              " for this collection");
        }
      }
    }
  }
  log.info("Returning CreateReplica command.");
  return new CreateReplica(collection, shard, node, replicaType, coreName, coreNodeName);
}
 

@Override
public List<ReplicaPosition> assign(SolrCloudManager solrCloudManager, AssignRequest assignRequest) throws Assign.AssignmentException, IOException, InterruptedException {
  if (assignRequest.numTlogReplicas + assignRequest.numPullReplicas != 0) {
    throw new Assign.AssignmentException(
        Replica.Type.TLOG + " or " + Replica.Type.PULL + " replica types not supported with placement rules or cluster policies");
  }

  Map<String, Integer> shardVsReplicaCount = new HashMap<>();
  for (String shard : assignRequest.shardNames) shardVsReplicaCount.put(shard, assignRequest.numNrtReplicas);

  Map<String, Map<String, Integer>> shardVsNodes = new LinkedHashMap<>();
  DocCollection docCollection = solrCloudManager.getClusterStateProvider().getClusterState().getCollectionOrNull(assignRequest.collectionName);
  if (docCollection != null) {
    for (Slice slice : docCollection.getSlices()) {
      LinkedHashMap<String, Integer> n = new LinkedHashMap<>();
      shardVsNodes.put(slice.getName(), n);
      for (Replica replica : slice.getReplicas()) {
        Integer count = n.get(replica.getNodeName());
        if (count == null) count = 0;
        n.put(replica.getNodeName(), ++count);
      }
    }
  }

  List<String> nodesList = assignRequest.nodes == null ? new ArrayList<>(clusterState.getLiveNodes()) : assignRequest.nodes;

  ReplicaAssigner replicaAssigner = new ReplicaAssigner(rules,
      shardVsReplicaCount,
      snitches,
      shardVsNodes,
      nodesList,
      solrCloudManager, clusterState);

  Map<ReplicaPosition, String> nodeMappings = replicaAssigner.getNodeMappings();
  return nodeMappings.entrySet().stream()
      .map(e -> new ReplicaPosition(e.getKey().shard, e.getKey().index, e.getKey().type, e.getValue()))
      .collect(Collectors.toList());
}
 

long getNumDocs(CloudSolrClient client) throws Exception {
  String collectionName = client.getDefaultCollection();
  DocCollection collection = client.getZkStateReader().getClusterState().getCollection(collectionName);
  Collection<Slice> slices = collection.getSlices();

  long totCount = 0;
  for (Slice slice : slices) {
    if (!slice.getState().equals(Slice.State.ACTIVE)) continue;
    long lastReplicaCount = -1;
    for (Replica replica : slice.getReplicas()) {
      SolrClient replicaClient = getHttpSolrClient(replica.getBaseUrl() + "/" + replica.getCoreName());
      long numFound = 0;
      try {
        numFound = replicaClient.query(params("q", "*:*", "distrib", "false")).getResults().getNumFound();
        log.info("Replica count={} for {}", numFound, replica);
      } finally {
        replicaClient.close();
      }
      if (lastReplicaCount >= 0) {
        assertEquals("Replica doc count for " + replica, lastReplicaCount, numFound);
      }
      lastReplicaCount = numFound;
    }
    totCount += lastReplicaCount;
  }


  long cloudClientDocs = client.query(new SolrQuery("*:*")).getResults().getNumFound();
  assertEquals("Sum of shard count should equal distrib query doc count", totCount, cloudClientDocs);
  return totCount;
}
 

private List<String> getShardUrls() throws IOException {
  try {
    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

    Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    List<String> baseUrls = new ArrayList<>();
    for(Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
          shuffler.add(replica);
        }
      }

      Collections.shuffle(shuffler, new Random());
      Replica rep = shuffler.get(0);
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      baseUrls.add(url);
    }

    return baseUrls;

  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

private void waitForCoreNodeName(CoreDescriptor descriptor) {
  int retryCount = 320;
  log.debug("look for our core node name");
  while (retryCount-- > 0) {
    final DocCollection docCollection = zkStateReader.getClusterState()
        .getCollectionOrNull(descriptor.getCloudDescriptor().getCollectionName());
    if (docCollection != null && docCollection.getSlicesMap() != null) {
      final Map<String, Slice> slicesMap = docCollection.getSlicesMap();
      for (Slice slice : slicesMap.values()) {
        for (Replica replica : slice.getReplicas()) {
          // TODO: for really large clusters, we could 'index' on this

          String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
          String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);

          String msgNodeName = getNodeName();
          String msgCore = descriptor.getName();

          if (msgNodeName.equals(nodeName) && core.equals(msgCore)) {
            descriptor.getCloudDescriptor()
                .setCoreNodeName(replica.getName());
            getCoreContainer().getCoresLocator().persist(getCoreContainer(), descriptor);
            return;
          }
        }
      }
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }
  }
}
 

String waitForCoreNodeName(String collectionName, String msgNodeName, String msgCore) {
  int retryCount = 320;
  while (retryCount-- > 0) {
    final DocCollection docCollection = zkStateReader.getClusterState().getCollectionOrNull(collectionName);
    if (docCollection != null && docCollection.getSlicesMap() != null) {
      Map<String,Slice> slicesMap = docCollection.getSlicesMap();
      for (Slice slice : slicesMap.values()) {
        for (Replica replica : slice.getReplicas()) {
          // TODO: for really large clusters, we could 'index' on this

          String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
          String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);

          if (nodeName.equals(msgNodeName) && core.equals(msgCore)) {
            return replica.getName();
          }
        }
      }
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }
  }
  throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find coreNodeName");
}
 

private String getShardId(DocCollection collection, String coreNodeName) {
  if (collection == null) return null;
  Map<String,Slice> slices = collection.getSlicesMap();
  if (slices != null) {
    for (Slice slice : slices.values()) {
      for (Replica replica : slice.getReplicas()) {
        String cnn = replica.getName();
        if (coreNodeName.equals(cnn)) {
          return slice.getName();
        }
      }
    }
  }
  return null;
}
 

public static String getRandomServer(CloudSolrClient cloudClient, String collName) {
  DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection(collName);
  List<String> urls = new ArrayList<>();
  for (Slice slice : coll.getSlices()) {
    for (Replica replica : slice.getReplicas())
      urls.add(""+replica.get(ZkStateReader.BASE_URL_PROP) + "/"+replica.get(ZkStateReader.CORE_NAME_PROP));
  }
  return urls.get(random().nextInt(urls.size()));
}
 

/**
 * Get a (reproducibly) random replica from a {@link Slice}
 */
protected static Replica getRandomReplica(Slice slice) {
  List<Replica> replicas = new ArrayList<>(slice.getReplicas());
  if (replicas.size() == 0)
    fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString());
  Collections.shuffle(replicas, random());
  return replicas.get(0);
}
 

@ProcessElement
public void process(@Element Read spec, OutputReceiver<Read> out) throws IOException {
  ConnectionConfiguration connectionConfig = spec.getConnectionConfiguration();
  try (AuthorizedSolrClient<CloudSolrClient> client = connectionConfig.createClient()) {
    String collection = spec.getCollection();
    final ClusterState clusterState = AuthorizedSolrClient.getClusterState(client);
    DocCollection docCollection = clusterState.getCollection(collection);
    for (Slice slice : docCollection.getSlices()) {
      ArrayList<Replica> replicas = new ArrayList<>(slice.getReplicas());
      Collections.shuffle(replicas);
      // Load balancing by randomly picking an active replica
      Replica randomActiveReplica = null;
      for (Replica replica : replicas) {
        // We need to check both state of the replica and live nodes
        // to make sure that the replica is alive
        if (replica.getState() == Replica.State.ACTIVE
            && clusterState.getLiveNodes().contains(replica.getNodeName())) {
          randomActiveReplica = replica;
          break;
        }
      }
      // TODO in case of this replica goes inactive while the pipeline runs.
      // We should pick another active replica of this shard.
      checkState(
          randomActiveReplica != null,
          "Can not found an active replica for slice %s",
          slice.getName());
      out.output(spec.withReplicaInfo(ReplicaInfo.create(checkNotNull(randomActiveReplica))));
    }
  }
}
 

/**
 * Pick one replica from each shard to send the shard requests to.
 *
 * @param collection that is being queried
 * @throws IOException if an exception occurs while finding replicas
 */
protected void pickShards(String collection) throws IOException {
  try {

    ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();

    Slice[] slices = clusterState.getCollection(collection).getActiveSlicesArr();

    for(Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      List<Replica> shuffler = new ArrayList<>();
      for(Replica replica : replicas) {
        if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName()))
        shuffler.add(replica);
      }

      Collections.shuffle(shuffler, new Random());
      Replica rep = shuffler.get(0);
      ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
      String url = zkProps.getCoreUrl();
      replicaUrls.add(url);
    }
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

private String getCoreUrl(String collectionName,
                          String origCorename, ClusterState clusterState, List<Slice> slices,
                          boolean byCoreName, boolean activeReplicas) {
  String coreUrl;
  Set<String> liveNodes = clusterState.getLiveNodes();
  Collections.shuffle(slices, random);

  for (Slice slice : slices) {
    List<Replica> randomizedReplicas = new ArrayList<>(slice.getReplicas());
    Collections.shuffle(randomizedReplicas, random);

    for (Replica replica : randomizedReplicas) {
      if (!activeReplicas || (liveNodes.contains(replica.getNodeName())
          && replica.getState() == Replica.State.ACTIVE)) {

        if (byCoreName && !origCorename.equals(replica.getStr(CORE_NAME_PROP))) {
          // if it's by core name, make sure they match
          continue;
        }
        if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
          // don't count a local core
          continue;
        }

        if (origCorename != null) {
          coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
        } else {
          coreUrl = replica.getCoreUrl();
          if (coreUrl.endsWith("/")) {
            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
          }
        }

        return coreUrl;
      }
    }
  }
  return null;
}
 

private void queryWithPreferReplicaTypes(CloudSolrClient cloudClient,
                                         String preferReplicaTypes,
                                         boolean preferLocalShards,
                                         String collectionName)
    throws Exception
{
  SolrQuery qRequest = new SolrQuery("*:*");
  ModifiableSolrParams qParams = new ModifiableSolrParams();

  final List<String> preferredTypes = Arrays.asList(preferReplicaTypes.split("\\|"));
  StringBuilder rule = new StringBuilder();
  preferredTypes.forEach(type -> {
    if (rule.length() != 0) {
      rule.append(',');
    }
    rule.append(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE);
    rule.append(':');
    rule.append(type);
  });
  if (preferLocalShards) {
    if (rule.length() != 0) {
      rule.append(',');
    }
    rule.append(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION);
    rule.append(":local");
  }
  qParams.add(ShardParams.SHARDS_PREFERENCE, rule.toString());  
  qParams.add(ShardParams.SHARDS_INFO, "true");
  qRequest.add(qParams);

  // CloudSolrClient sends the request to some node.
  // And since all the nodes are hosting cores from all shards, the
  // distributed query formed by this node will select cores from the
  // local shards only
  QueryResponse qResponse = cloudClient.query(collectionName, qRequest);

  Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("Unable to obtain "+ShardParams.SHARDS_INFO, shardsInfo);

  Map<String, String> replicaTypeMap = new HashMap<String, String>();
  DocCollection collection = getCollectionState(collectionName);
  for (Slice slice : collection.getSlices()) {
    for (Replica replica : slice.getReplicas()) {
      String coreUrl = replica.getCoreUrl();
      // It seems replica reports its core URL with a trailing slash while shard
      // info returned from the query doesn't. Oh well.
      if (coreUrl.endsWith("/")) {
        coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
      }
      replicaTypeMap.put(coreUrl, replica.getType().toString());
    }
  }

  // Iterate over shards-info and check that replicas of correct type responded
  SimpleOrderedMap<?> shardsInfoMap = (SimpleOrderedMap<?>)shardsInfo;
  @SuppressWarnings({"unchecked"})
  Iterator<Map.Entry<String, ?>> itr = shardsInfoMap.asMap(100).entrySet().iterator();
  List<String> shardAddresses = new ArrayList<String>();
  while (itr.hasNext()) {
    Map.Entry<String, ?> e = itr.next();
    assertTrue("Did not find map-type value in "+ShardParams.SHARDS_INFO, e.getValue() instanceof Map);
    String shardAddress = (String)((Map)e.getValue()).get("shardAddress");
    assertNotNull(ShardParams.SHARDS_INFO+" did not return 'shardAddress' parameter", shardAddress);
    assertTrue(replicaTypeMap.containsKey(shardAddress));
    assertTrue(preferredTypes.indexOf(replicaTypeMap.get(shardAddress)) == 0);
    shardAddresses.add(shardAddress);
  }
  assertTrue("No responses", shardAddresses.size() > 0);
  if (log.isInfoEnabled()) {
    log.info("Shards giving the response: {}", Arrays.toString(shardAddresses.toArray()));
  }
}
 

private void invokeBulkCall(String  cacheName, List<String> errs,
                            @SuppressWarnings({"rawtypes"})Map val) throws Exception {

  String payload = "{" +
      "'set-property' : {'query.CACHENAME.size':'CACHEVAL1'," +
      "                  'query.CACHENAME.initialSize':'CACHEVAL2'}," +
      "'set-property': {'query.CACHENAME.autowarmCount' : 'CACHEVAL3'}" +
      "}";

  Set<String> errmessages = new HashSet<>();
  for(int i =1;i<2;i++){//make it  ahigher number
    RestTestHarness publisher = randomRestTestHarness(r);
    String response;
    String val1;
    String val2;
    String val3;
    try {
      payload = payload.replaceAll("CACHENAME" , cacheName);
      val1 = String.valueOf(10 * i + 1);
      payload = payload.replace("CACHEVAL1", val1);
      val2 = String.valueOf(10 * i + 2);
      payload = payload.replace("CACHEVAL2", val2);
      val3 = String.valueOf(10 * i + 3);
      payload = payload.replace("CACHEVAL3", val3);

      response = publisher.post("/config", SolrTestCaseJ4.json(payload));
    } finally {
      publisher.close();
    }
    
    @SuppressWarnings({"rawtypes"})
    Map map = (Map) Utils.fromJSONString(response);
    Object errors = map.get("errors");
    if(errors!= null){
      errs.add(new String(Utils.toJSON(errors), StandardCharsets.UTF_8));
      return;
    }

    DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection("collection1");
    List<String> urls = new ArrayList<>();
    for (Slice slice : coll.getSlices()) {
      for (Replica replica : slice.getReplicas())
        urls.add(""+replica.get(ZkStateReader.BASE_URL_PROP) + "/"+replica.get(ZkStateReader.CORE_NAME_PROP));
    }


    //get another node
    String url = urls.get(urls.size() - 1);

    long startTime = System.nanoTime();
    long maxTimeoutSeconds = 20;
    while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
      Thread.sleep(100);
      errmessages.clear();
      MapWriter respMap = getAsMap(url + "/config/overlay", cloudClient);
      MapWriter m = (MapWriter) respMap._get("overlay/props", null);
      if(m == null) {
        errmessages.add(StrUtils.formatString("overlay does not exist for cache: {0} , iteration: {1} response {2} ", cacheName, i, respMap.toString()));
        continue;
      }


      Object o = m._get(asList("query", cacheName, "size"), null);
      if(!val1.equals(o.toString())) errmessages.add(StrUtils.formatString("'size' property not set, expected = {0}, actual {1}", val1, o));

      o = m._get(asList("query", cacheName, "initialSize"), null);
      if(!val2.equals(o.toString())) errmessages.add(StrUtils.formatString("'initialSize' property not set, expected = {0}, actual {1}", val2, o));

      o = m._get(asList("query", cacheName, "autowarmCount"), null);
      if(!val3.equals(o.toString())) errmessages.add(StrUtils.formatString("'autowarmCount' property not set, expected = {0}, actual {1}", val3, o));
      if(errmessages.isEmpty()) break;
    }
    if(!errmessages.isEmpty()) {
      errs.addAll(errmessages);
      return;
    }
  }

}