Java Code Examples for org.apache.solr.common.cloud.DocRouter#partitionRange()

@SuppressWarnings({"unchecked"})
public ZkWriteCommand createCollection(ClusterState clusterState, ZkNodeProps message) {
  String cName = message.getStr(NAME);
  log.debug("building a new cName: {}", cName);
  if (clusterState.hasCollection(cName)) {
    log.warn("Collection {} already exists. exit", cName);
    return ZkStateWriter.NO_OP;
  }

  Map<String, Object> routerSpec = DocRouter.getRouterSpec(message);
  String routerName = routerSpec.get(NAME) == null ? DocRouter.DEFAULT_NAME : (String) routerSpec.get(NAME);
  DocRouter router = DocRouter.getDocRouter(routerName);

  Object messageShardsObj = message.get("shards");

  Map<String, Slice> slices;
  if (messageShardsObj instanceof Map) { // we are being explicitly told the slice data (e.g. coll restore)
    slices = Slice.loadAllFromMap(cName, (Map<String, Object>)messageShardsObj);
  } else {
    List<String> shardNames = new ArrayList<>();

    if (router instanceof ImplicitDocRouter) {
      getShardNames(shardNames, message.getStr("shards", DocRouter.DEFAULT_NAME));
    } else {
      int numShards = message.getInt(ZkStateReader.NUM_SHARDS_PROP, -1);
      if (numShards < 1)
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "numShards is a required parameter for 'compositeId' router");
      getShardNames(numShards, shardNames);
    }
    List<DocRouter.Range> ranges = router.partitionRange(shardNames.size(), router.fullRange());//maybe null

    slices = new LinkedHashMap<>();
    for (int i = 0; i < shardNames.size(); i++) {
      String sliceName = shardNames.get(i);

      Map<String, Object> sliceProps = new LinkedHashMap<>(1);
      sliceProps.put(Slice.RANGE, ranges == null ? null : ranges.get(i));

      slices.put(sliceName, new Slice(sliceName, null, sliceProps,cName));
    }
  }

  Map<String, Object> collectionProps = new HashMap<>();

  for (Map.Entry<String, Object> e : OverseerCollectionMessageHandler.COLLECTION_PROPS_AND_DEFAULTS.entrySet()) {
    Object val = message.get(e.getKey());
    if (val == null) {
      val = OverseerCollectionMessageHandler.COLLECTION_PROPS_AND_DEFAULTS.get(e.getKey());
    }
    if (val != null) collectionProps.put(e.getKey(), val);
  }
  collectionProps.put(DocCollection.DOC_ROUTER, routerSpec);

  if (message.getStr("fromApi") == null) {
    collectionProps.put("autoCreated", "true");
  }

  DocCollection newCollection = new DocCollection(cName, slices, collectionProps, router, -1);

  return new ZkWriteCommand(cName, newCollection);
}
 

public void splitByRouteFieldTest() throws Exception  {
  log.info("Starting testSplitWithRouteField");
  String collectionName = "routeFieldColl";
  int numShards = 4;
  int replicationFactor = 2;
  int maxShardsPerNode = (((numShards * replicationFactor) / getCommonCloudSolrClient()
      .getZkStateReader().getClusterState().getLiveNodes().size())) + 1;

  HashMap<String, List<Integer>> collectionInfos = new HashMap<>();
  String shard_fld = "shard_s";
  try (CloudSolrClient client = createCloudClient(null)) {
    Map<String, Object> props = Utils.makeMap(
        REPLICATION_FACTOR, replicationFactor,
        MAX_SHARDS_PER_NODE, maxShardsPerNode,
        OverseerCollectionMessageHandler.NUM_SLICES, numShards,
        "router.field", shard_fld);

    createCollection(collectionInfos, collectionName, props, client);
  }

  List<Integer> list = collectionInfos.get(collectionName);
  checkForCollection(collectionName, list, null);

  waitForRecoveriesToFinish(false);

  String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName);

  try (HttpSolrClient collectionClient = getHttpSolrClient(url)) {

    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    final DocRouter router = clusterState.getCollection(collectionName).getRouter();
    Slice shard1 = clusterState.getCollection(collectionName).getSlice(SHARD1);
    DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange();
    final List<DocRouter.Range> ranges = router.partitionRange(2, shard1Range);
    final int[] docCounts = new int[ranges.size()];

    for (int i = 100; i <= 200; i++) {
      String shardKey = "" + (char) ('a' + (i % 26)); // See comment in ShardRoutingTest for hash distribution

      collectionClient.add(getDoc(id, i, "n_ti", i, shard_fld, shardKey));
      int idx = getHashRangeIdx(router, ranges, shardKey);
      if (idx != -1) {
        docCounts[idx]++;
      }
    }

    for (int i = 0; i < docCounts.length; i++) {
      int docCount = docCounts[i];
      log.info("Shard shard1_{} docCount = {}", i, docCount);
    }

    collectionClient.commit();

    trySplit(collectionName, null, SHARD1, 3);

    waitForRecoveriesToFinish(collectionName, false);

    assertEquals(docCounts[0], collectionClient.query(new SolrQuery("*:*").setParam("shards", "shard1_0")).getResults().getNumFound());
    assertEquals(docCounts[1], collectionClient.query(new SolrQuery("*:*").setParam("shards", "shard1_1")).getResults().getNumFound());
  }
}
 

/***
  public void testPrintHashCodes() throws Exception {
   // from negative to positive, the upper bits of the hash ranges should be
   // shard1: 11
   // shard2: 10
   // shard3: 00
   // shard4: 01

   String[] highBitsToShard = {"shard3","shard4","shard1","shard2"};


   for (int i = 0; i<26; i++) {
      String id  = new String(Character.toChars('a'+i));
      int hash = hash(id);
      System.out.println("hash of " + id + " is " + Integer.toHexString(hash) + " high bits=" + (hash>>>30)
          + " shard="+highBitsToShard[hash>>>30]);
    }
  }
  ***/



@SuppressWarnings({"unchecked"})
DocCollection createCollection(int nSlices, DocRouter router) {
  List<Range> ranges = router.partitionRange(nSlices, router.fullRange());

  Map<String,Slice> slices = new HashMap<>();
  for (int i=0; i<ranges.size(); i++) {
    Range range = ranges.get(i);
    Slice slice = new Slice("shard"+(i+1), null, map("range",range), "collections1");
    slices.put(slice.getName(), slice);
  }

  DocCollection coll = new DocCollection("collection1", slices, null, router);
  return coll;
}