org.apache.solr.util.RTimer Java Examples

int doSetGen(int iter, Query q) throws Exception {
  SolrQueryRequest req = lrf.makeRequest();

  SolrIndexSearcher searcher = req.getSearcher();

  final RTimer timer = new RTimer();

  int ret = 0;
  for (int i=0; i<iter; i++) {
    DocSet set = searcher.getDocSetNC(q, null);
    ret += set.size();
  }

  double elapsed = timer.getTime();
  System.out.println("ret="+ret+ " time="+elapsed+" throughput="+iter*1000/(elapsed+1));

  req.close();
  assertTrue(ret>0);  // make sure we did some work
  return ret;
}
 

protected void sendCommitWithRetry(Replica replica) throws Exception {
  String replicaCoreUrl = replica.getCoreUrl();
  log.info("Sending commit request to: {}", replicaCoreUrl);
  final RTimer timer = new RTimer();
  try (HttpSolrClient client = getHttpSolrClient(replicaCoreUrl)) {
    try {
      client.commit();

      if (log.isInfoEnabled()) {
        log.info("Sent commit request to {} OK, took {}ms", replicaCoreUrl, timer.getTime());
      }
    } catch (Exception exc) {
      Throwable rootCause = SolrException.getRootCause(exc);
      if (rootCause instanceof NoHttpResponseException) {
        log.warn("No HTTP response from sending commit request to {}; will re-try after waiting 3 seconds", replicaCoreUrl);
        Thread.sleep(3000);
        client.commit();
        log.info("Second attempt at sending commit to {} succeeded", replicaCoreUrl);
      } else {
        throw exc;
      }
    }
  }
}
 

/** Opens a new realtime searcher and returns it's (possibly cached) fingerprint */
public static IndexFingerprint getFingerprint(SolrCore core, long maxVersion) throws IOException {
  RTimer timer = new RTimer();
  core.getUpdateHandler().getUpdateLog().openRealtimeSearcher();
  RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().uhandler.core.getRealtimeSearcher();
  try {
    IndexFingerprint f = newestSearcher.get().getIndexFingerprint(maxVersion);
    final double duration = timer.stop();
    log.info("IndexFingerprint millis:{} result:{}",duration, f);
    return f;
  } finally {
    if (newestSearcher != null) {
      newestSearcher.decref();
    }
  }
}
 

/** Process the request with the facet context settings, a parameter-object. */
final Object process(FacetContext fcontext) throws IOException {
  @SuppressWarnings("rawtypes")
  FacetProcessor facetProcessor = createFacetProcessor(fcontext);

  FacetDebugInfo debugInfo = fcontext.getDebugInfo();
  if (debugInfo == null) {
    facetProcessor.process();
  } else {
    if (fcontext.filter != null) {
      debugInfo.setFilter(fcontext.filter.toString());
    }
    debugInfo.setReqDescription(getFacetDescription());
    debugInfo.setProcessor(facetProcessor.getClass().getSimpleName());
    debugInfo.putInfoItem("domainSize", (long) fcontext.base.size());
    RTimer timer = new RTimer();
    try {
      facetProcessor.process();
    }finally {
      debugInfo.setElapse((long) timer.getTime());
    }
  }

  return facetProcessor.getResponse(); 
}
 

int doListGen(int iter, Query q, List<Query> filt, boolean cacheQuery, boolean cacheFilt) throws Exception {
  SolrQueryRequest req = lrf.makeRequest();

  SolrIndexSearcher searcher = req.getSearcher();

  final RTimer timer = new RTimer();

  int ret = 0;
  for (int i=0; i<iter; i++) {
    DocList l = searcher.getDocList(q, filt, (Sort)null, 0, 10, (cacheQuery?0:SolrIndexSearcher.NO_CHECK_QCACHE)|(cacheFilt?0:SolrIndexSearcher.NO_CHECK_FILTERCACHE) );
    ret += l.matches();
  }

  double elapsed = timer.getTime();
  System.out.println("ret="+ret+ " time="+elapsed+" throughput="+iter*1000/(elapsed+1));

  req.close();
  assertTrue(ret>0);  // make sure we did some work
  return ret;
}
 

public static void postAndCheck(CloudSolrClient cloudClient, String baseUrl, String blobName, ByteBuffer bytes, int count) throws Exception {
  postData(cloudClient, baseUrl, blobName, bytes);

  String url;
  MapWriter map = null;
  final RTimer timer = new RTimer();
  int i = 0;
  for (; i < 150; i++) {//15 secs
    url = baseUrl + "/.system/blob/" + blobName;
    map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
    String numFound = map._getStr(asList("response", "numFound"),null);
    if (!("" + count).equals(numFound)) {
      Thread.sleep(100);
      continue;
    }

    assertEquals("" + bytes.limit(), map._getStr("response/docs[0]/size",null));
    return;
  }
  fail(StrUtils.formatString("Could not successfully add blob after {0} attempts. Expecting {1} items. time elapsed {2}  output  for url is {3}",
      i, count, timer.getTime(), map.toString()));
}
 

public void doCommitPerf() throws Exception {

    try (HttpSolrClient client = getHttpSolrClient("http://127.0.0.1:8983/solr")) {

      final RTimer timer = new RTimer();

      for (int i = 0; i < 10000; i++) {
        SolrInputDocument doc = new SolrInputDocument();
        doc.addField("id", Integer.toString(i % 13));
        client.add(doc);
        client.commit(true, true, true);
      }

      System.out.println("TIME: " + timer.getTime());
    }

  }
 

private void initHostname() {
  if (null != System.getProperty(PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP, null)) {
    log.info("Resolving canonical hostname for local host prevented due to '{}' sysprop",
             PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP);
    hostname = null;
    return;
  }
  
  RTimer timer = new RTimer();
  try {
    InetAddress addr = InetAddress.getLocalHost();
    hostname = addr.getCanonicalHostName();
  } catch (Exception e) {
    log.warn("Unable to resolve canonical hostname for local host, possible DNS misconfiguration. SET THE '{}' {}"
        , PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP
        , " sysprop to true on startup to prevent future lookups if DNS can not be fixed.", e);
    hostname = null;
    return;
  }
  timer.stop();
  
  if (15000D < timer.getTime()) {
    String readableTime = String.format(Locale.ROOT, "%.3f", (timer.getTime() / 1000));
    log.warn("Resolving canonical hostname for local host took {} seconds, possible DNS misconfiguration. Set the '{}' {}"
        , readableTime, PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP,
        " sysprop to true on startup to prevent future lookups if DNS can not be fixed.");
  }
}
 

ClusterState waitForNewShard(String collectionName, String sliceName) throws KeeperException, InterruptedException {
  log.debug("Waiting for slice {} of collection {} to be available", sliceName, collectionName);
  RTimer timer = new RTimer();
  int retryCount = 320;
  while (retryCount-- > 0) {
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection = clusterState.getCollection(collectionName);

    if (collection == null) {
      throw new SolrException(ErrorCode.SERVER_ERROR,
          "Unable to find collection: " + collectionName + " in clusterstate");
    }
    Slice slice = collection.getSlice(sliceName);
    if (slice != null) {
      if (log.isDebugEnabled()) {
        log.debug("Waited for {}ms for slice {} of collection {} to be available",
            timer.getTime(), sliceName, collectionName);
      }
      return clusterState;
    }
    Thread.sleep(1000);
  }
  throw new SolrException(ErrorCode.SERVER_ERROR,
      "Could not find new slice " + sliceName + " in collection " + collectionName
          + " even after waiting for " + timer.getTime() + "ms"
  );
}
 

@Override
public Boolean call() throws Exception {
  final RTimer timer = new RTimer();
  int attempts = 0;
  try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) {
    // eventually, this loop will get killed by the ExecutorService's timeout
    while (true) {
      try {
        long timeElapsed = (long) timer.getTime() / 1000;
        if (timeElapsed >= maxWait) {
          return false;
        }
        log.info("Time elapsed : {} secs, maxWait {}", timeElapsed, maxWait);
        Thread.sleep(100);
        NamedList<Object> resp = solr.httpUriRequest(this).future.get();
        if (resp != null) {
          @SuppressWarnings({"rawtypes"})
          Map m = (Map) resp.get(ZNODEVER);
          if (m != null) {
            remoteVersion = (Number) m.get(prop);
            if (remoteVersion != null && remoteVersion.intValue() >= expectedZkVersion) break;
          }
        }

        attempts++;
        if (log.isInfoEnabled()) {
          log.info(formatString("Could not get expectedVersion {0} from {1} for prop {2}   after {3} attempts", expectedZkVersion, coreUrl, prop, attempts));
        }
      } catch (Exception e) {
        if (e instanceof InterruptedException) {
          break; // stop looping
        } else {
          log.warn("Failed to get /schema/zkversion from {} due to: ", coreUrl, e);
        }
      }
    }
  }
  return true;
}
 

@Test
@Monster("Only useful to verify the performance of serialization+ deserialization")
// ant -Dtestcase=SolrExampleBinaryTest -Dtests.method=testQueryPerf -Dtests.monster=true test
public void testQueryPerf() throws Exception {
  HttpSolrClient client = (HttpSolrClient) getSolrClient();
  client.deleteByQuery("*:*");
  client.commit();
  ArrayList<SolrInputDocument> docs = new ArrayList<>();
  int id = 0;
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "apple", "cat", "a", "inStock", true, "popularity", 12, "price", .017));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "lg", "cat", "a", "inStock", false, "popularity", 13, "price", 16.04));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "samsung", "cat", "a", "inStock", true, "popularity", 14, "price", 12.34));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "lg", "cat", "b", "inStock", false, "popularity", 24, "price", 51.39));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "nokia", "cat", "b", "inStock", true, "popularity", 28, "price", 131.39));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "ztc", "cat", "a", "inStock", false, "popularity", 32));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "htc", "cat", "a", "inStock", true, "popularity", 31, "price", 131.39));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "apple", "cat", "b", "inStock", false, "popularity", 36));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "lg", "cat", "b", "inStock", true, "popularity", 37, "price", 1.39));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "ztc", "cat", "b", "inStock", false, "popularity", 38, "price", 47.98));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "ztc", "cat", "b", "inStock", true, "popularity", -38));
  docs.add(makeTestDoc("id", id++, "cat", "b")); // something not matching all fields
  client.add(docs);
  client.commit();
  //this sets the cache
  QueryResponse rsp = getSolrClient().query(new SolrQuery("*:*").setRows(20));

  RTimer timer = new RTimer();
  int count = 10000;
  log.info("Started perf test....");
  for(int i=0;i< count; i++){
    rsp = getSolrClient().query(new SolrQuery("*:*").setRows(20));
  }

  if (log.isInfoEnabled()) {
    log.info("time taken to execute {} queries is {} ms", count, timer.getTime());
  }

}
 

/**
 * Used to seed all version buckets with the max value of the version field in the index.
 */
protected Long seedBucketsWithHighestVersion(SolrIndexSearcher newSearcher, VersionInfo versions) {
  Long highestVersion = null;
  final RTimer timer = new RTimer();

  try (RecentUpdates recentUpdates = getRecentUpdates()) {
    long maxVersionFromRecent = recentUpdates.getMaxRecentVersion();
    long maxVersionFromIndex = versions.getMaxVersionFromIndex(newSearcher);

    long maxVersion = Math.max(maxVersionFromIndex, maxVersionFromRecent);
    if (maxVersion == 0L) {
      maxVersion = versions.getNewClock();
      log.info("Could not find max version in index or recent updates, using new clock {}", maxVersion);
    }

    // seed all version buckets with the highest value from recent and index
    versions.seedBucketsWithHighestVersion(maxVersion);

    highestVersion = maxVersion;
  } catch (IOException ioExc) {
    log.warn("Failed to determine the max value of the version field due to: ", ioExc);
  }

  if (debug) {
    log.debug("Took {}ms to seed version buckets with highest version {}",
        timer.getTime(), highestVersion);
  }

  return highestVersion;
}
 

public static void main(String[] args) throws Exception {
  // String addr = "http://odin.local:80/solr";
  // String addr = "http://odin.local:8983/solr";
  String addr = "http://127.0.0.1:8983/solr";

  int i = 0;
  final int nDocs = Integer.parseInt(args[i++]);
  final int nProducers = Integer.parseInt(args[i++]);
  final int nConnections = Integer.parseInt(args[i++]);
  final int maxSleep = Integer.parseInt(args[i++]);

  ConcurrentUpdateSolrClient concurrentClient = null;

  // server = concurrentClient = new ConcurrentUpdateSolrServer(addr,32,8);
  client = concurrentClient = getConcurrentUpdateSolrClient(addr,64,nConnections);

  client.deleteByQuery("*:*");
  client.commit();

  final RTimer timer = new RTimer();

  final int docsPerThread = nDocs / nProducers;

  Thread[] threads = new Thread[nProducers];

  for (int threadNum = 0; threadNum<nProducers; threadNum++) {
    final int base = threadNum * docsPerThread;

    threads[threadNum] = new Thread("add-thread"+i) {
      @Override
      public void run(){
        try {
          indexDocs(base, docsPerThread, maxSleep);
        } catch (Exception e) {
          System.out.println("###############################CAUGHT EXCEPTION");
          e.printStackTrace();
          ex = e;
        }
      }
    };
    threads[threadNum].start();
  }

  // optional: wait for commit?

  for (int threadNum = 0; threadNum<nProducers; threadNum++) {
    threads[threadNum].join();
  }

  if (concurrentClient != null) {
    concurrentClient.blockUntilFinished();
  }

  double elapsed = timer.getTime();
  System.out.println("time="+elapsed + " throughput="+(nDocs*1000/elapsed) + " Exception="+ex);

  // should server threads be marked as daemon?
  // need a server.close()!!!
}
 

public void setTimer(RTimer timer) {
  this.timer = timer;
}
 

protected void waitToSeeReplicasActive(String testCollectionName, String shardId, Set<String> replicasToCheck, int maxWaitSecs) throws Exception {
  final RTimer timer = new RTimer();

  ZkStateReader zkr = cloudClient.getZkStateReader();
  zkr.forceUpdateCollection(testCollectionName);
  ClusterState cs = zkr.getClusterState();
  boolean allReplicasUp = false;
  long waitMs = 0L;
  long maxWaitMs = maxWaitSecs * 1000L;
  while (waitMs < maxWaitMs && !allReplicasUp) {
    cs = cloudClient.getZkStateReader().getClusterState();
    assertNotNull(cs);
    final DocCollection docCollection = cs.getCollectionOrNull(testCollectionName);
    assertNotNull(docCollection);
    Slice shard = docCollection.getSlice(shardId);
    assertNotNull("No Slice for "+shardId, shard);
    allReplicasUp = true; // assume true

    // wait to see all replicas are "active"
    for (Replica replica : shard.getReplicas()) {
      if (!replicasToCheck.contains(replica.getName()))
        continue;

      final Replica.State state = replica.getState();
      if (state != Replica.State.ACTIVE) {
        if (log.isInfoEnabled()) {
          log.info("Replica {} is currently {}", replica.getName(), state);
        }
        allReplicasUp = false;
      }
    }

    if (!allReplicasUp) {
      try {
        Thread.sleep(200L);
      } catch (Exception ignoreMe) {}
      waitMs += 200L;
    }
  } // end while

  if (!allReplicasUp)
    fail("Didn't see replicas "+ replicasToCheck +
        " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo(testCollectionName));

  if (log.isInfoEnabled()) {
    log.info("Took {} ms to see replicas [{}] become active.", timer.getTime(), replicasToCheck);
  }
}
 

@Override
public RTimer getRequestTimer() {
	return request.getRequestTimer();
}
 

@Override
public RTimer getRequestTimer() {
	return request.getRequestTimer();
}
 

public void testIndexingPerf() throws IOException {
  int iter=1000;
  String iterS = System.getProperty("iter");
  if (iterS != null) iter=Integer.parseInt(iterS);
  boolean overwrite = Boolean.parseBoolean(System.getProperty("overwrite","false"));
  String doc = System.getProperty("doc");
  if (doc != null) {
    StrUtils.splitSmart(doc,",",true);
  }


  SolrQueryRequest req = lrf.makeRequest();
  UpdateHandler updateHandler = req.getCore().getUpdateHandler();
  String field = "textgap";

  String[] fields = {field,"simple"
          ,field,"test"
          ,field,"how now brown cow"
          ,field,"what's that?"
          ,field,"radical!"
          ,field,"what's all this about, anyway?"
          ,field,"just how fast is this text indexing?"
  };


/***
  String[] fields = {
          "a_i","1"
          ,"b_i","2"
          ,"c_i","3"
          ,"d_i","4"
          ,"e_i","5"
          ,"f_i","6"
          ,"g_i","7"
          ,"h_i","8"
          ,"i_i","9"
          ,"j_i","0"
          ,"k_i","0"
  };
 ***/

  final RTimer timer = new RTimer();

  AddUpdateCommand add = new AddUpdateCommand(req);
  add.overwrite = overwrite;

  for (int i=0; i<iter; i++) {
    add.clear();
    add.solrDoc = new SolrInputDocument();
    add.solrDoc.addField("id", Integer.toString(i));
    for (int j=0; j<fields.length; j+=2) {
      String f = fields[j];
      String val = fields[j+1];
      add.solrDoc.addField(f, val);
    }
    updateHandler.addDoc(add);
  }
  if (log.isInfoEnabled()) {
    log.info("doc={}", Arrays.toString(fields));
  }
  double elapsed = timer.getTime();
  if (log.isInfoEnabled()) {
    log.info("iter={} time={} throughput={}", iter, elapsed, ((long) iter * 1000) / elapsed);
  }

  //discard all the changes
  updateHandler.rollback(new RollbackUpdateCommand(req));

  req.close();
}
 

/**
 * Ensures that a searcher is registered for the given core and if not, waits until one is registered
 */
private static void ensureRegisteredSearcher(SolrCore core) throws InterruptedException {
  if (!core.getSolrConfig().useColdSearcher) {
    RefCounted<SolrIndexSearcher> registeredSearcher = core.getRegisteredSearcher();
    if (registeredSearcher != null) {
      if (log.isDebugEnabled()) {
        log.debug("Found a registered searcher: {} for core: {}", registeredSearcher.get(), core);
      }
      registeredSearcher.decref();
    } else  {
      @SuppressWarnings({"rawtypes"})
      Future[] waitSearcher = new Future[1];
      if (log.isInfoEnabled()) {
        log.info("No registered searcher found for core: {}, waiting until a searcher is registered before publishing as active", core.getName());
      }
      final RTimer timer = new RTimer();
      RefCounted<SolrIndexSearcher> searcher = null;
      try {
        searcher = core.getSearcher(false, true, waitSearcher, true);
        boolean success = true;
        if (waitSearcher[0] != null)  {
          if (log.isDebugEnabled()) {
            log.debug("Waiting for first searcher of core {}, id: {} to be registered", core.getName(), core);
          }
          try {
            waitSearcher[0].get();
          } catch (ExecutionException e) {
            log.warn("Wait for a searcher to be registered for core {}, id: {} failed due to: {}", core.getName(), core, e, e);
            success = false;
          }
        }
        if (success)  {
          if (searcher == null) {
            // should never happen
            if (log.isDebugEnabled()) {
              log.debug("Did not find a searcher even after the future callback for core: {}, id: {}!!!", core.getName(), core);
            }
          } else  {
            if (log.isInfoEnabled()) {
              log.info("Found a registered searcher: {}, took: {} ms for core: {}, id: {}", searcher.get(), timer.getTime(), core.getName(), core);
            }
          }
        }
      } finally {
        if (searcher != null) {
          searcher.decref();
        }
      }
    }
    RefCounted<SolrIndexSearcher> newestSearcher = core.getNewestSearcher(false);
    if (newestSearcher != null) {
      if (log.isDebugEnabled()) {
        log.debug("Found newest searcher: {} for core: {}, id: {}", newestSearcher.get(), core.getName(), core);
      }
      newestSearcher.decref();
    }
  }
}
 

public RTimer getTimer() {
  return timer;
}
 

@SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats")
private void markReplicationStart() {
  replicationTimer = new RTimer();
  replicationStartTimeStamp = new Date();
}
 

/**
 *   Returns a list of range counts sorted by the range lower bound, using the indexed "id" field (i.e. the terms are full IDs, not just prefixes)
 */
static Collection<RangeCount> getHashHistogramFromId(SolrIndexSearcher searcher, String idField, DocRouter router, DocCollection collection) throws IOException {
  RTimer timer = new RTimer();

  TreeMap<DocRouter.Range, RangeCount> counts = new TreeMap<>();

  Terms terms = MultiTerms.getTerms(searcher.getIndexReader(), idField);
  if (terms == null) {
    return counts.values();
  }

  int numPrefixes = 0;
  int numCollisions = 0;
  long sumBuckets = 0;


  byte sep = (byte) CompositeIdRouter.SEPARATOR.charAt(0);
  TermsEnum termsEnum = terms.iterator();
  BytesRef currPrefix = new BytesRef();  // prefix of the previous "id" term
  int bucketCount = 0; // count of the number of docs in the current bucket

  // We're going to iterate over all terms, so do the minimum amount of work per term.
  // Terms are sorted, so all terms sharing a prefix will be grouped together.  The extra work
  // is really just limited to stepping over all the terms in the id field.
  for (;;) {
    BytesRef term = termsEnum.next();

    // compare to current prefix bucket and see if this new term shares the same prefix
    if (term != null && term.length >= currPrefix.length && currPrefix.length > 0) {
      if (StringHelper.startsWith(term, currPrefix)) {
        bucketCount++;  // use 1 since we are dealing with unique ids
        continue;
      }
    }

    // At this point the prefix did not match, so if we had a bucket we were working on, record it.
    if (currPrefix.length > 0) {
      numPrefixes++;
      sumBuckets += bucketCount;
      String currPrefixStr = currPrefix.utf8ToString();
      DocRouter.Range range = router.getSearchRangeSingle(currPrefixStr, null, collection);

      RangeCount rangeCount = new RangeCount(range, bucketCount);
      bucketCount = 0;

      RangeCount prev = counts.put(rangeCount.range, rangeCount);
      if (prev != null) {
        // we hit a hash collision, so add the buckets together.
        rangeCount.count += prev.count;
        numCollisions++;
      }
    }

    // if the current term is null, we ran out of values
    if (term == null) break;

    // find the new prefix (if any)

    // resize if needed
    if (currPrefix.length < term.length) {
      currPrefix.bytes = new byte[term.length+10];
    }

    // Copy the bytes up to and including the separator, and set the length if the separator is found.
    // If there was no separator, then length remains 0 and it's the indicator that we have no prefix bucket
    currPrefix.length = 0;
    for (int i=0; i<term.length; i++) {
      byte b = term.bytes[i + term.offset];
      currPrefix.bytes[i] = b;
      if (b == sep) {
        currPrefix.length = i + 1;
        bucketCount++;
        break;
      }
    }
  }

  if (log.isInfoEnabled()) {
    log.info("Split histogram from idField {}: ms={}, numBuckets={} sumBuckets={} numPrefixes={} numCollisions={}"
        , idField, timer.getTime(), counts.size(), sumBuckets, numPrefixes, numCollisions);
  }

  return counts.values();
}
 

static Collection<RangeCount> getHashHistogram(SolrIndexSearcher searcher, String prefixField, DocRouter router, DocCollection collection) throws IOException {
  RTimer timer = new RTimer();
  TreeMap<DocRouter.Range,RangeCount> counts = new TreeMap<>();

  Terms terms = MultiTerms.getTerms(searcher.getIndexReader(), prefixField);
  if (terms == null) {
    return counts.values();
  }

  int numPrefixes = 0;
  int numTriLevel = 0;
  int numCollisions = 0;
  long sumBuckets = 0;

  TermsEnum termsEnum = terms.iterator();
  BytesRef term;
  while ((term = termsEnum.next()) != null) {
    numPrefixes++;

    String termStr = term.utf8ToString();
    int firstSep = termStr.indexOf(CompositeIdRouter.SEPARATOR);
    // truncate to first separator since we don't support multiple levels currently
    // NOTE: this does not currently work for tri-level composite ids since the number of bits allocated to the first ID is 16 for a 2 part id
    // and 8 for a 3 part id!
    if (firstSep != termStr.length()-1 && firstSep > 0) {
      numTriLevel++;
      termStr = termStr.substring(0, firstSep+1);
    }

    DocRouter.Range range = router.getSearchRangeSingle(termStr, null, collection);
    int numDocs = termsEnum.docFreq();
    sumBuckets += numDocs;

    RangeCount rangeCount = new RangeCount(range, numDocs);

    RangeCount prev = counts.put(rangeCount.range, rangeCount);
    if (prev != null) {
      // we hit a hash collision or truncated a prefix to first level, so add the buckets together.
      rangeCount.count += prev.count;
      numCollisions++;
    }
  }

  if (log.isInfoEnabled()) {
    log.info("Split histogram: ms={}, numBuckets={} sumBuckets={} numPrefixes={} numTriLevel={} numCollisions={}"
        , timer.getTime(), counts.size(), sumBuckets, numPrefixes, numTriLevel, numCollisions);
  }

  return counts.values();
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  if (filter == null) {
    boolean debug = rb != null && rb.isDebug();
    RTimer timer = (debug ? new RTimer() : null);
    resultSet = getDocSet();
    if (timer != null) timer.stop();

    if (debug) {
      SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<>();
      dbg.add("time", (long) timer.getTime());
      dbg.add("fromSetSize", fromSetSize);  // the input
      dbg.add("toSetSize", resultSet.size());    // the output

      dbg.add("fromTermCount", fromTermCount);
      dbg.add("fromTermTotalDf", fromTermTotalDf);
      dbg.add("fromTermDirectCount", fromTermDirectCount);
      dbg.add("fromTermHits", fromTermHits);
      dbg.add("fromTermHitsTotalDf", fromTermHitsTotalDf);
      dbg.add("toTermHits", toTermHits);
      dbg.add("toTermHitsTotalDf", toTermHitsTotalDf);
      dbg.add("toTermDirectCount", toTermDirectCount);
      dbg.add("smallSetsDeferred", smallSetsDeferred);
      dbg.add("toSetDocsAdded", resultListDocs);

      // TODO: perhaps synchronize  addDebug in the future...
      rb.addDebug(dbg, "join", JoinQuery.this.toString());
    }

    filter = resultSet.getTopFilter();
  }

  // Although this set only includes live docs, other filters can be pushed down to queries.
  DocIdSet readerSet = filter.getDocIdSet(context, null);
  if (readerSet == null) {
    return null;
  }
  DocIdSetIterator readerSetIterator = readerSet.iterator();
  if (readerSetIterator == null) {
    return null;
  }
  return new ConstantScoreScorer(this, score(), scoreMode, readerSetIterator);
}
 

public MessageIterator(Folder folder, int batchSize) {
  super();
  
  try {
    this.folder = folder;
    this.batchSize = batchSize;
    SearchTerm st = getSearchTerm();
    
    log.info("SearchTerm={}", st);
    
    if (st != null || folder instanceof GmailFolder) {
      doBatching = false;
      // Searching can still take a while even though we're only pulling
      // envelopes; unless you're using gmail server-side filter, which is
      // fast
      if (log.isInfoEnabled()) {
        log.info("Searching folder {} for messages", folder.getName());
      }
      final RTimer searchTimer = new RTimer();

      // If using GMail, speed up the envelope processing by doing a
      // server-side
      // search for messages occurring on or after the fetch date (at
      // midnight),
      // which reduces the number of envelopes we need to pull from the
      // server
      // to apply the precise DateTerm filter; GMail server-side search has
      // date
      // granularity only but the local filters are also applied
                
      if (folder instanceof GmailFolder && fetchMailsSince != null) {
        String afterCrit = "after:" + afterFmt.format(fetchMailsSince);
        log.info("Added server-side gmail filter: {}", afterCrit);
        Message[] afterMessages = folder.search(new GmailRawSearchTerm(
            afterCrit));

        if (log.isInfoEnabled()) {
          log.info("GMail server-side filter found {} messages received {} in folder {}"
              , afterMessages.length, afterCrit, folder.getName());
        }
        
        // now pass in the server-side filtered messages to the local filter
        messagesInCurBatch = folder.search((st != null ? st : this), afterMessages);
      } else {
        messagesInCurBatch = folder.search(st);
      }          
      totalInFolder = messagesInCurBatch.length;
      folder.fetch(messagesInCurBatch, fp);
      current = 0;
      if (log.isInfoEnabled()) {
        log.info("Total messages : {}", totalInFolder);
        log.info("Search criteria applied. Batching disabled. Took {} (ms)", searchTimer.getTime()); // logOk
      }
    } else {
      totalInFolder = folder.getMessageCount();
      log.info("Total messages : {}", totalInFolder);
      getNextBatch(batchSize, folder);
    }
  } catch (MessagingException e) {
    throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
        "Message retreival failed", e);
  }
}
 

public void startTheMonkey(boolean killLeaders, final int roundPauseUpperLimit) {
  if (!MONKEY_ENABLED) {
    monkeyLog("The Monkey is disabled and will not start");
    return;
  }
  monkeyLog("starting");
  
  
  if (chaosRandom.nextBoolean()) {
    monkeyLog("Jetty will not commit on close");
    TestInjection.skipIndexWriterCommitOnClose = true;
  }

  this.aggressivelyKillLeaders = killLeaders;
  runTimer = new RTimer();
  // TODO: when kill leaders is on, lets kill a higher percentage of leaders
  
  stop = false;
  monkeyThread = new Thread() {

    @Override
    public void run() {
      while (!stop) {
        try {
  
          Thread.sleep(chaosRandom.nextInt(roundPauseUpperLimit));

          causeSomeChaos();
          
        } catch (InterruptedException e) {
          //
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
      monkeyLog("finished");
      monkeyLog("I ran for " + runTimer.getTime() / 1000 + "s. I stopped " + stops + " and I started " + starts
          + ". I also expired " + expires.get() + " and caused " + connloss
          + " connection losses");
    }
  };
  monkeyThread.start();
}
 

@SuppressWarnings({"unchecked"})
public static void doDecodePerf(String[] args) throws Exception {
  int arg=0;
  int nThreads = Integer.parseInt(args[arg++]);
  int nBuffers = Integer.parseInt(args[arg++]);
  final long iter = Long.parseLong(args[arg++]);
  int cacheSz = Integer.parseInt(args[arg++]);

  Random r = new Random(0);

  final byte[][] buffers = new byte[nBuffers][];

  for (int bufnum=0; bufnum<nBuffers; bufnum++) {
    SolrDocument sdoc = new SolrDocument();
    sdoc.put("id", "my_id_" + bufnum);
    sdoc.put("author", str(r, 10 + r.nextInt(10)));
    sdoc.put("address", str(r, 20 + r.nextInt(20)));
    sdoc.put("license", str(r, 10));
    sdoc.put("title", str(r, 5 + r.nextInt(10)));
    sdoc.put("modified_dt", r.nextInt(1000000));
    sdoc.put("creation_dt", r.nextInt(1000000));
    sdoc.put("birthdate_dt", r.nextInt(1000000));
    sdoc.put("clean", r.nextBoolean());
    sdoc.put("dirty", r.nextBoolean());
    sdoc.put("employed", r.nextBoolean());
    sdoc.put("priority", r.nextInt(100));
    sdoc.put("dependents", r.nextInt(6));
    sdoc.put("level", r.nextInt(101));
    sdoc.put("education_level", r.nextInt(10));
    // higher level of reuse for string values
    sdoc.put("state", "S"+r.nextInt(50));
    sdoc.put("country", "Country"+r.nextInt(20));
    sdoc.put("some_boolean", ""+r.nextBoolean());
    sdoc.put("another_boolean", ""+r.nextBoolean());

    buffers[bufnum] = getBytes(sdoc);
  }

  int ret = 0;
  final RTimer timer = new RTimer();
  @SuppressWarnings({"rawtypes"})
  ConcurrentLRUCache underlyingCache = cacheSz > 0 ? new ConcurrentLRUCache<>(cacheSz,cacheSz-cacheSz/10,cacheSz,cacheSz/10,false,true,null) : null;  // the cache in the first version of the patch was 10000,9000,10000,1000,false,true,null
  final JavaBinCodec.StringCache stringCache = underlyingCache==null ? null : new JavaBinCodec.StringCache(underlyingCache);
  if (nThreads <= 0) {
    ret += doDecode(buffers, iter, stringCache);
  } else {
    runInThreads(nThreads, () -> {
      try {
        doDecode(buffers, iter, stringCache);
      } catch (IOException e) {
        e.printStackTrace();
      }
    });
  }

  long n = iter * Math.max(1,nThreads);
  System.out.println("ret=" + ret + " THROUGHPUT=" + (n*1000 / timer.getTime()));
  if (underlyingCache != null) System.out.println("cache: hits=" + underlyingCache.getStats().getCumulativeHits() + " lookups=" + underlyingCache.getStats().getCumulativeLookups() + " size=" + underlyingCache.getStats().getCurrentSize());
}