org.apache.lucene.search.FieldDoc Java Examples

/** tests expression referring to another expression */
public void testExpressionRefersToExpression() throws Exception {
  Expression expr1 = JavascriptCompiler.compile("_score");
  Expression expr2 = JavascriptCompiler.compile("2*expr1");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES);
  bindings.add("expr1", expr1);
  
  Sort sort = new Sort(expr2.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = 2*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

@Override
public void process(ResponseBuilder rb, ShardRequest shardRequest) {
  boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
  ShardResponse srsp = shardRequest.responses.get(0);
  SolrDocumentList docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response");
  String uniqueIdFieldName = rb.req.getSchema().getUniqueKeyField().getName();

  for (SolrDocument doc : docs) {
    Object id = doc.getFieldValue(uniqueIdFieldName).toString();
    ShardDoc shardDoc = rb.resultIds.get(id);
    FieldDoc fieldDoc = (FieldDoc) shardDoc;
    if (shardDoc != null) {
      if (returnScores && !Float.isNaN(fieldDoc.score)) {
          doc.setField("score", fieldDoc.score);
      }
      rb.retrievedDocuments.put(id, doc);
    }
  }
}
 

@Nullable
@Override
public Row apply(@Nullable ScoreDoc input) {
    if (input == null) {
        return null;
    }
    FieldDoc fieldDoc = (FieldDoc) input;
    scorer.score(fieldDoc.score);
    for (OrderByCollectorExpression orderByCollectorExpression : orderByCollectorExpressions) {
        orderByCollectorExpression.setNextFieldDoc(fieldDoc);
    }
    List<LeafReaderContext> leaves = indexReader.leaves();
    int readerIndex = ReaderUtil.subIndex(fieldDoc.doc, leaves);
    LeafReaderContext subReaderContext = leaves.get(readerIndex);
    int subDoc = fieldDoc.doc - subReaderContext.docBase;
    for (LuceneCollectorExpression<?> expression : expressions) {
        expression.setNextReader(subReaderContext);
        expression.setNextDocId(subDoc);
    }
    return inputRow;
}
 

@Nullable
@Override
public Row apply(@Nullable ScoreDoc input) {
    onScoreDoc.run();
    if (input == null) {
        return null;
    }
    FieldDoc fieldDoc = (FieldDoc) input;
    scorer.score(fieldDoc.score);
    for (int i = 0; i < orderByCollectorExpressions.size(); i++) {
        orderByCollectorExpressions.get(i).setNextFieldDoc(fieldDoc);
    }
    List<LeafReaderContext> leaves = indexReader.leaves();
    int readerIndex = ReaderUtil.subIndex(fieldDoc.doc, leaves);
    LeafReaderContext subReaderContext = leaves.get(readerIndex);
    int subDoc = fieldDoc.doc - subReaderContext.docBase;
    for (LuceneCollectorExpression<?> expression : expressions) {
        try {
            expression.setNextReader(subReaderContext);
            expression.setNextDocId(subDoc);
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }
    return inputRow;
}
 

private void doTestLotsOfBindings(int n) throws Exception {
  SimpleBindings bindings = new SimpleBindings();    
  StringBuilder sb = new StringBuilder();
  for (int i = 0; i < n; i++) {
    if (i > 0) {
      sb.append("+");
    }
    sb.append("x" + i);
    bindings.add("x" + i, DoubleValuesSource.SCORES);
  }
  
  Expression expr = JavascriptCompiler.compile(sb.toString());
  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = n*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

/** Uses variables with $ */
public void testDollarVariable() throws Exception {
  Expression expr = JavascriptCompiler.compile("$0+$score");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("$0", DoubleValuesSource.SCORES);
  bindings.add("$score", DoubleValuesSource.SCORES);

  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = 2*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

/** tests same binding used more than once in an expression */
public void testTwoOfSameBinding() throws Exception {
  Expression expr = JavascriptCompiler.compile("_score + _score");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES);
  
  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = 2*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

/** tests the returned sort values are correct */
public void testSortValues() throws Exception {
  Expression expr = JavascriptCompiler.compile("sqrt(_score)");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES);

  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = (float) Math.sqrt(d.score);
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

private LindenHit parseSort(ScoreDoc hit, LindenHit lindenHit) {
  if (request.isSetSort()) {
    Map<String, String> fieldMap = new HashMap<>();
    for (int i = 0; i < request.getSort().getFields().size(); ++i) {
      LindenSortField field = request.getSort().getFields().get(i);
      if (field.type == LindenSortType.SCORE || field.type == LindenSortType.DISTANCE) {
        continue;
      }
      Object value = ((FieldDoc) hit).fields[i];
      if (value == null) {
        continue;
      }
      if (field.type == LindenSortType.STRING) {
        fieldMap.put(field.getName(), ((BytesRef) value).utf8ToString());
      } else {
        fieldMap.put(field.getName(), value.toString());
      }
    }
    lindenHit.setFields(fieldMap);
  }
  return lindenHit;
}
 

private List<LindenHit> parseLindenHits(ScoreDoc[] hits) throws IOException {
  List<LindenHit> lindenHits = new ArrayList<>();
  String idFieldName = config.getSchema().getId();
  for (ScoreDoc hit : hits) {
    LindenHit lindenHit = new LindenHit();
    if (Double.isNaN(hit.score)) {
      // get score for cluster result merge
      if (sortScoreFieldPos != -1) {
        lindenHit.setScore(Double.valueOf(((FieldDoc) hit).fields[sortScoreFieldPos].toString()));
      } else {
        lindenHit.setScore(1);
      }
    } else {
      lindenHit.setScore(hit.score);
    }
    String id = LindenUtil.getFieldStringValue(leaves, hit.doc, idFieldName);
    lindenHit.setId(id);
    lindenHit = this.parseSpatial(hit.doc, lindenHit);
    lindenHit = this.parseSort(hit, lindenHit);
    lindenHit = this.parseSource(hit.doc, lindenHit);
    lindenHit = this.parseExplain(hit.doc, lindenHit);
    lindenHits.add(lindenHit);
  }
  lindenHits = this.parseSnippets(lindenHits, hits);
  return lindenHits;
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeLong(id);
    out.writeVInt(size);
    for (int i = 0; i < size; i++) {
        out.writeVInt(docIds[i]);
    }
    if (lastEmittedDoc == null) {
        out.writeByte((byte) 0);
    } else if (lastEmittedDoc instanceof FieldDoc) {
        out.writeByte((byte) 1);
        Lucene.writeFieldDoc(out, (FieldDoc) lastEmittedDoc);
    } else {
        out.writeByte((byte) 2);
        Lucene.writeScoreDoc(out, lastEmittedDoc);
    }
}
 

@Test
public void testSearchAfterWithSystemColumn() {
    Reference sysColReference =
        new Reference(
            new ReferenceIdent(
                new RelationName(Schemas.DOC_SCHEMA_NAME, "table"),
                DocSysColumns.SCORE), RowGranularity.DOC, DataTypes.FLOAT, null, null
        );

    OrderBy orderBy = new OrderBy(ImmutableList.of(sysColReference, REFERENCE),
        new boolean[]{false, false},
        new boolean[]{false, false});

    FieldDoc lastCollected = new FieldDoc(0, 0, new Object[]{2L});

    OptimizeQueryForSearchAfter queryForSearchAfter = new OptimizeQueryForSearchAfter(
        orderBy, mock(QueryShardContext.class), name -> valueFieldType);
    Query nextPageQuery = queryForSearchAfter.apply(lastCollected);

    // returns null which leads to reuse of old query without paging optimization
    assertNull(nextPageQuery);
}
 

public void testNearestNeighborWithAllDeletedDocs() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, getIndexWriterConfig());
  Document doc = new Document();
  doc.add(new FloatPoint("point", 40.0f, 50.0f));
  doc.add(new StringField("id", "0", Field.Store.YES));
  w.addDocument(doc);
  doc = new Document();
  doc.add(new FloatPoint("point", 45.0f, 55.0f));
  doc.add(new StringField("id", "1", Field.Store.YES));
  w.addDocument(doc);

  DirectoryReader r = w.getReader();
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps with its own points impl:
  IndexSearcher s = newSearcher(r, false);
  FieldDoc hit = (FieldDoc)FloatPointNearestNeighbor.nearest(s, "point", 1, 40.0f, 50.0f).scoreDocs[0];
  assertEquals("0", r.document(hit.doc).getField("id").stringValue());
  r.close();

  w.deleteDocuments(new Term("id", "0"));
  w.deleteDocuments(new Term("id", "1"));
  r = w.getReader();
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps with its own points impl:
  s = newSearcher(r, false);
  assertEquals(0, FloatPointNearestNeighbor.nearest(s, "point", 1, 40.0f, 50.0f).scoreDocs.length);
  r.close();
  w.close();
  dir.close();
}
 

private Query query(FieldDoc lastDoc) {
    Query optimizedQuery = searchAfterQueryOptimize.apply(lastDoc);
    if (optimizedQuery == null) {
        return this.query;
    }
    BooleanQuery.Builder searchAfterQuery = new BooleanQuery.Builder();
    searchAfterQuery.add(this.query, BooleanClause.Occur.MUST);
    searchAfterQuery.add(optimizedQuery, BooleanClause.Occur.MUST_NOT);
    return searchAfterQuery.build();
}
 

private KeyIterable<ShardId, Row> scoreDocToIterable(ScoreDoc[] scoreDocs) {
    exhausted = scoreDocs.length < batchSize;
    if (scoreDocs.length > 0) {
        lastDoc = (FieldDoc) scoreDocs[scoreDocs.length - 1];
    }
    return new KeyIterable<>(shardId(), Iterables.transform(Arrays.asList(scoreDocs), rowFunction));
}
 

public LuceneOrderedDocCollector(ShardId shardId,
                                 IndexSearcher searcher,
                                 Query query,
                                 Float minScore,
                                 boolean doDocsScores,
                                 int batchSize,
                                 RamAccounting ramAccounting,
                                 CollectorContext collectorContext,
                                 Function<FieldDoc, Query> searchAfterQueryOptimize,
                                 Sort sort,
                                 List<? extends Input<?>> inputs,
                                 Collection<? extends LuceneCollectorExpression<?>> expressions) {
    super(shardId);
    this.searcher = searcher;
    this.query = query;
    this.minScore = minScore;
    this.doDocsScores = doDocsScores;
    this.ramAccounting = ramAccounting;
    // We don't want to pre-allocate for more records than what can possible be returned
    // (+1) to make sure `exhausted` is set to `true` if all records match on the first `collect` call.
    this.batchSize = Math.min(batchSize, searcher.getIndexReader().numDocs() + 1);
    this.collectorContext = collectorContext;
    this.searchAfterQueryOptimize = searchAfterQueryOptimize;
    this.sort = sort;
    this.scorer = new DummyScorer();
    this.expressions = expressions;
    this.rowFunction = new ScoreDocRowFunction(
        searcher.getIndexReader(),
        inputs,
        expressions,
        scorer,
        this::raiseIfKilled
    );
}
 

public BlurFieldCollector(Sort sort, int numHitsToCollect, FieldDoc after, boolean runSlow, AtomicBoolean running) {
  _sort = sort;
  _numHitsToCollect = numHitsToCollect;
  _after = after;
  _runSlow = runSlow;
  _running = running;
}
 

@Test
public void testNextPageQueryWithLastCollectedNullValue() {
    FieldDoc fieldDoc = new FieldDoc(1, 0, new Object[]{null});
    OrderBy orderBy = new OrderBy(Collections.singletonList(REFERENCE), new boolean[]{false}, new boolean[]{false});

    OptimizeQueryForSearchAfter queryForSearchAfter = new OptimizeQueryForSearchAfter(
        orderBy, mock(QueryShardContext.class), name -> valueFieldType);

    queryForSearchAfter.apply(fieldDoc);
}
 

public BlurResultIterableSearcher(AtomicBoolean running, Query query, String table, String shard,
    IndexSearcherCloseable searcher, Selector selector, boolean closeSearcher, boolean runSlow, int fetchCount,
    int maxHeapPerRowFetch, TableContext context, Sort sort, DeepPagingCache deepPagingCache) throws BlurException {
  _sort = sort;
  _running = running;
  _query = query;
  _shard = shard;
  _searcher = searcher;
  _closeSearcher = closeSearcher;
  _runSlow = runSlow;
  _fetchCount = fetchCount;
  _iterablePaging = new IterablePaging(_running, _searcher, _query, _fetchCount, _totalHitsRef, _progressRef,
      _runSlow, _sort, deepPagingCache);
  _iteratorConverter = new IteratorConverter<ScoreDoc, BlurResult, BlurException>(_iterablePaging.iterator(),
      new Converter<ScoreDoc, BlurResult, BlurException>() {
        @Override
        public BlurResult convert(ScoreDoc scoreDoc) throws BlurException {
          String resolveId = resolveId(scoreDoc);
          if (_sort == null) {
            return new BlurResult(resolveId, scoreDoc.score, null, null);
          } else {
            FieldDoc fieldDoc = (FieldDoc) scoreDoc;
            return new BlurResult(resolveId, scoreDoc.score, null, BlurUtil.convertToSortFields(fieldDoc.fields));
          }
        }
      });
  _shardInfo.put(_shard, (long) _totalHitsRef.totalHits());
}
 

private static void testNumericDVSort(IndexSearcher s) throws IOException {
  // Confirm we can sort by the new DV field:
  TopDocs hits = s.search(new MatchAllDocsQuery(), 100, new Sort(new SortField("number", SortField.Type.LONG)));
  long last = Long.MIN_VALUE;
  for(ScoreDoc scoreDoc : hits.scoreDocs) {
    long value = Long.parseLong(s.doc(scoreDoc.doc).get("text").split(" ")[1]);
    assertTrue(value >= last);
    assertEquals(value, ((Long) ((FieldDoc) scoreDoc).fields[0]).longValue());
    last = value;
  }
}
 

public void testMultipleUpdatesSameDoc() throws Exception {

    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    
    conf.setMaxBufferedDocs(3); // small number of docs, so use a tiny maxBufferedDocs

    IndexWriter writer = new IndexWriter(dir, conf);

    writer.updateDocument       (new Term("id","doc-1"), doc(1, 1000000000L ));
    writer.updateNumericDocValue(new Term("id","doc-1"), "val", 1000001111L );
    writer.updateDocument       (new Term("id","doc-2"), doc(2, 2000000000L ));
    writer.updateDocument       (new Term("id","doc-2"), doc(2, 2222222222L ));
    writer.updateNumericDocValue(new Term("id","doc-1"), "val", 1111111111L );

    final DirectoryReader reader;
    if (random().nextBoolean()) {
      writer.commit();
      reader = DirectoryReader.open(dir);
    } else {
      reader = DirectoryReader.open(writer);
    }
    final IndexSearcher searcher = new IndexSearcher(reader);
    TopFieldDocs td;
    
    td = searcher.search(new TermQuery(new Term("id", "doc-1")), 1, 
                         new Sort(new SortField("val", SortField.Type.LONG)));
    assertEquals("doc-1 missing?", 1, td.scoreDocs.length);
    assertEquals("doc-1 value", 1111111111L, ((FieldDoc)td.scoreDocs[0]).fields[0]);
    
    td = searcher.search(new TermQuery(new Term("id", "doc-2")), 1, 
                        new Sort(new SortField("val", SortField.Type.LONG)));
    assertEquals("doc-2 missing?", 1, td.scoreDocs.length);
    assertEquals("doc-2 value", 2222222222L, ((FieldDoc)td.scoreDocs[0]).fields[0]);
    
    IOUtils.close(reader, writer, dir);
  }
 

/**
 * Assert that the given {@link TopDocs} have the same top docs and consistent hit counts.
 */
public static void assertConsistent(TopDocs expected, TopDocs actual) {
  Assert.assertEquals("wrong total hits", expected.totalHits.value == 0, actual.totalHits.value == 0);
  if (expected.totalHits.relation == TotalHits.Relation.EQUAL_TO) {
    if (actual.totalHits.relation == TotalHits.Relation.EQUAL_TO) {
      Assert.assertEquals("wrong total hits", expected.totalHits.value, actual.totalHits.value);
    } else {
      Assert.assertTrue("wrong total hits", expected.totalHits.value >= actual.totalHits.value);
    }
  } else if (actual.totalHits.relation == TotalHits.Relation.EQUAL_TO) {
    Assert.assertTrue("wrong total hits", expected.totalHits.value <= actual.totalHits.value);
  }
  Assert.assertEquals("wrong hit count", expected.scoreDocs.length, actual.scoreDocs.length);
  for(int hitIDX=0;hitIDX<expected.scoreDocs.length;hitIDX++) {
    final ScoreDoc expectedSD = expected.scoreDocs[hitIDX];
    final ScoreDoc actualSD = actual.scoreDocs[hitIDX];
    Assert.assertEquals("wrong hit docID", expectedSD.doc, actualSD.doc);
    Assert.assertEquals("wrong hit score", expectedSD.score, actualSD.score, 0.0);
    if (expectedSD instanceof FieldDoc) {
      Assert.assertTrue(actualSD instanceof FieldDoc);
      Assert.assertArrayEquals("wrong sort field values",
                          ((FieldDoc) expectedSD).fields,
                          ((FieldDoc) actualSD).fields);
    } else {
      Assert.assertFalse(actualSD instanceof FieldDoc);
    }
  }
}
 

public static TopFieldDocs nearest(IndexSearcher searcher, String field, int topN, float... origin) throws IOException {
  if (topN < 1) {
    throw new IllegalArgumentException("topN must be at least 1; got " + topN);
  }
  if (field == null) {
    throw new IllegalArgumentException("field must not be null");
  }
  if (searcher == null) {
    throw new IllegalArgumentException("searcher must not be null");
  }
  List<BKDReader> readers = new ArrayList<>();
  List<Integer> docBases = new ArrayList<>();
  List<Bits> liveDocs = new ArrayList<>();
  int totalHits = 0;
  for (LeafReaderContext leaf : searcher.getIndexReader().leaves()) {
    PointValues points = leaf.reader().getPointValues(field);
    if (points != null) {
      if (points instanceof BKDReader == false) {
        throw new IllegalArgumentException("can only run on Lucene60PointsReader points implementation, but got " + points);
      }
      totalHits += points.getDocCount();
      readers.add((BKDReader)points);
      docBases.add(leaf.docBase);
      liveDocs.add(leaf.reader().getLiveDocs());
    }
  }

  NearestHit[] hits = nearest(readers, liveDocs, docBases, topN, origin);

  // Convert to TopFieldDocs:
  ScoreDoc[] scoreDocs = new ScoreDoc[hits.length];
  for(int i=0;i<hits.length;i++) {
    NearestHit hit = hits[i];
    scoreDocs[i] = new FieldDoc(hit.docID, 0.0f, new Object[] { (float)Math.sqrt(hit.distanceSquared) });
  }
  return new TopFieldDocs(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), scoreDocs, null);
}
 

public void testSortGroups() throws IOException {

    Shard shard = new Shard();
    indexRandomDocs(shard.writer);
    IndexSearcher searcher = shard.getIndexSearcher();

    String[] query = new String[]{ "foo", "bar", "baz" };
    Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));

    GroupingSearch grouper = new GroupingSearch(getGroupSelector());
    grouper.setGroupDocsLimit(10);
    Sort sort = new Sort(new SortField("sort1", SortField.Type.STRING), new SortField("sort2", SortField.Type.LONG));
    grouper.setGroupSort(sort);
    TopGroups<T> topGroups = grouper.search(searcher, topLevel, 0, 5);
    TopDocs topDoc = searcher.search(topLevel, 1, sort);
    for (int i = 0; i < topGroups.groups.length; i++) {
      // We're sorting the groups by a defined Sort, but each group itself should be ordered
      // by doc relevance, and should be equal to the results of a top-level query filtered
      // by the group value
      Query filtered = new BooleanQuery.Builder()
          .add(topLevel, BooleanClause.Occur.MUST)
          .add(filterQuery(topGroups.groups[i].groupValue), BooleanClause.Occur.FILTER)
          .build();
      TopDocs td = searcher.search(filtered, 10);
      assertScoreDocsEquals(topGroups.groups[i].scoreDocs, td.scoreDocs);
      // The top group should have sort values equal to the sort values of the top doc of
      // a top-level search sorted by the same Sort; subsequent groups should have sort values
      // that compare lower than their predecessor.
      if (i > 0) {
        assertSortsBefore(topGroups.groups[i - 1], topGroups.groups[i]);
      } else {
        assertArrayEquals(((FieldDoc)topDoc.scoreDocs[0]).fields, topGroups.groups[0].groupSortValues);
      }
    }

    shard.close();
  }
 

protected static void assertFieldDocsEquals(ScoreDoc[] expected, ScoreDoc[] actual) {
  assertEquals(expected.length, actual.length);
  for (int i = 0; i < expected.length; i++) {
    assertEquals(expected[i].doc, actual[i].doc);
    FieldDoc e = (FieldDoc) expected[i];
    FieldDoc a = (FieldDoc) actual[i];
    assertArrayEquals(e.fields, a.fields);
  }
}
 

public void testTopLevelSort() throws IOException {

    Shard shard = new Shard();
    indexRandomDocs(shard.writer);
    IndexSearcher searcher = shard.getIndexSearcher();

    Sort sort = new Sort(new SortField("length", SortField.Type.LONG));

    Query blockEndQuery = new TermQuery(new Term("blockEnd", "true"));
    GroupingSearch grouper = new GroupingSearch(blockEndQuery);
    grouper.setGroupDocsLimit(10);
    grouper.setGroupSort(sort);     // groups returned sorted by length, chapters within group sorted by relevancy

    Query topLevel = new TermQuery(new Term("text", "grandmother"));
    TopGroups<?> tg = grouper.search(searcher, topLevel, 0, 5);

    // The sort value of the top doc in the top group should be the same as the sort value
    // of the top result from the same search done with no grouping
    TopDocs topDoc = searcher.search(topLevel, 1, sort);
    assertEquals(((FieldDoc)topDoc.scoreDocs[0]).fields[0], tg.groups[0].groupSortValues[0]);

    for (int i = 0; i < tg.groups.length; i++) {
      String bookName = searcher.doc(tg.groups[i].scoreDocs[0].doc).get("book");
      // The contents of each group should be equal to the results of a search for
      // that group alone, sorted by score
      Query filtered = new BooleanQuery.Builder()
          .add(topLevel, BooleanClause.Occur.MUST)
          .add(new TermQuery(new Term("book", bookName)), BooleanClause.Occur.FILTER)
          .build();
      TopDocs td = searcher.search(filtered, 10);
      assertScoreDocsEquals(td.scoreDocs, tg.groups[i].scoreDocs);
      if (i > 1) {
        assertSortsBefore(tg.groups[i - 1], tg.groups[i]);
      }
    }

    shard.close();

  }
 

@Override
public InternalAggregation buildAggregation(long owningBucketOrdinal) {
    TopDocsAndLeafCollector topDocsCollector = topDocsCollectors.get(owningBucketOrdinal);
    final InternalTopHits topHits;
    if (topDocsCollector == null) {
        topHits = buildEmptyAggregation();
    } else {
        final TopDocs topDocs = topDocsCollector.topLevelCollector.topDocs();

        subSearchContext.queryResult().topDocs(topDocs);
        int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
        for (int i = 0; i < topDocs.scoreDocs.length; i++) {
            docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
        }
        subSearchContext.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
        fetchPhase.execute(subSearchContext);
        FetchSearchResult fetchResult = subSearchContext.fetchResult();
        InternalSearchHit[] internalHits = fetchResult.fetchResult().hits().internalHits();
        for (int i = 0; i < internalHits.length; i++) {
            ScoreDoc scoreDoc = topDocs.scoreDocs[i];
            InternalSearchHit searchHitFields = internalHits[i];
            searchHitFields.shard(subSearchContext.shardTarget());
            searchHitFields.score(scoreDoc.score);
            if (scoreDoc instanceof FieldDoc) {
                FieldDoc fieldDoc = (FieldDoc) scoreDoc;
                searchHitFields.sortValues(fieldDoc.fields);
            }
        }
        topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(), pipelineAggregators(),
                metaData());
    }
    return topHits;
}
 

public static BlurFieldCollector create(Sort sort, int numHitsToCollect, FieldDoc after, boolean runSlow,
    AtomicBoolean running) {
  return new BlurFieldCollector(sort, numHitsToCollect, after, runSlow, running);
}
 

private void assertEquals(int[] docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean idvBasedImplsUsed) {
  if (expected == null) {
    assertNull(actual);
    return;
  }
  assertNotNull(actual);

  assertEquals("expected.groups.length != actual.groups.length", expected.groups.length, actual.groups.length);
  assertEquals("expected.totalHitCount != actual.totalHitCount", expected.totalHitCount, actual.totalHitCount);
  assertEquals("expected.totalGroupedHitCount != actual.totalGroupedHitCount", expected.totalGroupedHitCount, actual.totalGroupedHitCount);
  if (expected.totalGroupCount != null && verifyTotalGroupCount) {
    assertEquals("expected.totalGroupCount != actual.totalGroupCount", expected.totalGroupCount, actual.totalGroupCount);
  }

  for(int groupIDX=0;groupIDX<expected.groups.length;groupIDX++) {
    if (VERBOSE) {
      System.out.println("  check groupIDX=" + groupIDX);
    }
    final GroupDocs<BytesRef> expectedGroup = expected.groups[groupIDX];
    final GroupDocs<BytesRef> actualGroup = actual.groups[groupIDX];
    if (verifyGroupValues) {
      if (idvBasedImplsUsed) {
        if (actualGroup.groupValue.length == 0) {
          assertNull(expectedGroup.groupValue);
        } else {
          assertEquals(expectedGroup.groupValue, actualGroup.groupValue);
        }
      } else {
        assertEquals(expectedGroup.groupValue, actualGroup.groupValue);
      }

    }
    assertArrayEquals(expectedGroup.groupSortValues, actualGroup.groupSortValues);

    // TODO
    // assertEquals(expectedGroup.maxScore, actualGroup.maxScore);
    assertEquals(expectedGroup.totalHits.value, actualGroup.totalHits.value);

    final ScoreDoc[] expectedFDs = expectedGroup.scoreDocs;
    final ScoreDoc[] actualFDs = actualGroup.scoreDocs;

    assertEquals(expectedFDs.length, actualFDs.length);
    for(int docIDX=0;docIDX<expectedFDs.length;docIDX++) {
      final FieldDoc expectedFD = (FieldDoc) expectedFDs[docIDX];
      final FieldDoc actualFD = (FieldDoc) actualFDs[docIDX];
      //System.out.println("  actual doc=" + docIDtoID[actualFD.doc] + " score=" + actualFD.score);
      assertEquals(expectedFD.doc, docIDtoID[actualFD.doc]);
      assertArrayEquals(expectedFD.fields, actualFD.fields);
    }
  }
}
 

@Override
public Query apply(FieldDoc lastCollected) {
    BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
    for (int i = 0; i < orderBy.orderBySymbols().size(); i++) {
        Symbol order = orderBy.orderBySymbols().get(i);
        Object value = lastCollected.fields[i];
        if (order instanceof Reference) {
            final ColumnIdent columnIdent = ((Reference) order).column();
            if (columnIdent.isSystemColumn()) {
                // We can't optimize the initial query because the BooleanQuery
                // must not contain system columns.
                return null;
            }
            boolean nullsFirst = orderBy.nullsFirst()[i];
            value = value == null || value.equals(missingValues[i]) ? null : value;
            if (nullsFirst && value == null) {
                // no filter needed
                continue;
            }
            String columnName = columnIdent.fqn();
            MappedFieldType fieldType = requireNonNull(
                fieldTypeLookup.get(columnName), "Column must exist: " + columnName);

            Query orderQuery;
            // nulls already gone, so they should be excluded
            if (nullsFirst) {
                BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
                booleanQuery.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
                if (orderBy.reverseFlags()[i]) {
                    booleanQuery.add(fieldType.rangeQuery(null, value, false, true, null, null, queryShardContext), BooleanClause.Occur.MUST_NOT);
                } else {
                    booleanQuery.add(fieldType.rangeQuery(value, null, true, false, null, null, queryShardContext), BooleanClause.Occur.MUST_NOT);
                }
                orderQuery = booleanQuery.build();
            } else {
                if (orderBy.reverseFlags()[i]) {
                    orderQuery = fieldType.rangeQuery(value, null, false, false, null, null, queryShardContext);
                } else {
                    orderQuery = fieldType.rangeQuery(null, value, false, false, null, null, queryShardContext);
                }
            }
            queryBuilder.add(orderQuery, BooleanClause.Occur.MUST);
        }
    }
    BooleanQuery query = queryBuilder.build();
    if (query.clauses().size() > 0) {
        return query;
    } else {
        return null;
    }
}