org.apache.lucene.index.MultiDocValues Java Examples

@Override
public Query rewrite(IndexReader reader) throws IOException {
    if (getBoost() != 1.0F) {
        return super.rewrite(reader);
    }
    if (reader instanceof DirectoryReader) {
        String joinField = ParentFieldMapper.joinField(parentType);
        IndexSearcher indexSearcher = new IndexSearcher(reader);
        indexSearcher.setQueryCache(null);
        indexSearcher.setSimilarity(similarity);
        IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal((DirectoryReader) reader);
        MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType);
        return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren);
    } else {
        if (reader.leaves().isEmpty() && reader.numDocs() == 0) {
            // asserting reader passes down a MultiReader during rewrite which makes this
            // blow up since for this query to work we have to have a DirectoryReader otherwise
            // we can't load global ordinals - for this to work we simply check if the reader has no leaves
            // and rewrite to match nothing
            return new MatchNoDocsQuery();
        }
        throw new IllegalStateException("can't load global ordinals for reader of type: " + reader.getClass() + " must be a DirectoryReader");
    }
}
 

public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  groups = new LongObjectHashMap<>(numGroups);
  DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
  int group;
  while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    groups.put(group, getCollector(limit, sort));
  }

  this.collapsedSet = collapsedSet;
  this.groupBits = groupBits;
  this.docValues = docValues;
  if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
}
 

/** Does all the "real work" of tallying up the counts. */
private final void countAll() throws IOException {
  //System.out.println("ssdv count");

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  for(LeafReaderContext context : state.getReader().leaves()) {
    countOneSegment(ordinalMap, context.reader(), context.ord, null);
  }
}
 

public void testBasics() throws Exception {
  // sanity check of norms writer
  // TODO: generalize
  NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo");
  NumericDocValues barNorms = MultiDocValues.getNormValues(reader, "bar");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, fooNorms.nextDoc());
    assertEquals(i, barNorms.nextDoc());
    assertFalse(fooNorms.longValue() == barNorms.longValue());
  }

  // sanity check of searching
  TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
  assertTrue(foodocs.totalHits.value > 0);
  TopDocs bardocs = searcher.search(new TermQuery(new Term("bar", "brown")), 10);
  assertTrue(bardocs.totalHits.value > 0);
  assertTrue(foodocs.scoreDocs[0].score < bardocs.scoreDocs[0].score);
}
 

static DocumentField of(FieldInfo finfo, IndexableField field, IndexReader reader, int docId)
    throws IOException {

  Objects.requireNonNull(finfo);
  Objects.requireNonNull(reader);

  DocumentField dfield = new DocumentField();

  dfield.name = finfo.name;
  dfield.idxOptions = finfo.getIndexOptions();
  dfield.hasTermVectors = finfo.hasVectors();
  dfield.hasPayloads = finfo.hasPayloads();
  dfield.hasNorms = finfo.hasNorms();

  if (finfo.hasNorms()) {
    NumericDocValues norms = MultiDocValues.getNormValues(reader, finfo.name);
    if (norms.advanceExact(docId)) {
      dfield.norm = norms.longValue();
    }
  }

  dfield.dvType = finfo.getDocValuesType();

  dfield.pointDimensionCount = finfo.getPointDimensionCount();
  dfield.pointNumBytes = finfo.getPointNumBytes();

  if (field != null) {
    dfield.isStored = field.fieldType().stored();
    dfield.stringValue = field.stringValue();
    if (field.binaryValue() != null) {
      dfield.binaryValue = BytesRef.deepCopyOf(field.binaryValue());
    }
    dfield.numericValue = field.numericValue();
  }

  return dfield;
}
 

public StringValue(SortedDocValues globalDocValues, String field, IntComp comp)  {
  this.globalDocValues = globalDocValues;
  this.docValues = globalDocValues;
  if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) globalDocValues).mapping;
  }
  this.field = field;
  this.comp = comp;
  this.currentOrd = comp.resetValue();
  this.present = false;
}
 

@Override
public void resetIterators() throws IOException {
  super.resetIterators();
  topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf, null);
  if (topLevel instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordMap = ((MultiDocValues.MultiSortedSetDocValues)topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedSetDocValues)topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 

@Override
public void resetIterators() throws IOException {
  super.resetIterators();
  topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field, null);
  if (topLevel instanceof MultiDocValues.MultiSortedDocValues) {
    ordMap = ((MultiDocValues.MultiSortedDocValues)topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedDocValues)topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 

@Override
public void resetIterators() throws IOException {
  topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, field, null);
  nTerms = (int) topLevel.getValueCount();
  if (topLevel instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordMap = ((MultiDocValues.MultiSortedSetDocValues) topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedSetDocValues) topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 

@Override
public void resetIterators() throws IOException {
  super.resetIterators();
  topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field, null);
  nTerms = topLevel.getValueCount();
  if (topLevel instanceof MultiDocValues.MultiSortedDocValues) {
    ordMap = ((MultiDocValues.MultiSortedDocValues)topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedDocValues)topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 

private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception { 
  SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");
  for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {
    Document doc = is.doc(docID);
    String s = doc.getField("field").stringValue();
    boolean collatorAccepts = collate(collator, s, startPoint) >= 0 && collate(collator, s, endPoint) <= 0;
    assertEquals(docID, dvs.nextDoc());
    BytesRef br = dvs.binaryValue();
    boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
    assertEquals(startPoint + " <= " + s + " <= " + endPoint, collatorAccepts, luceneAccepts);
  }
}
 

private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception { 
  SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");
  for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {
    Document doc = is.doc(docID);
    String s = doc.getField("field").stringValue();
    boolean collatorAccepts = collator.compare(s, startPoint) >= 0 && collator.compare(s, endPoint) <= 0;
    assertEquals(docID, dvs.nextDoc());
    BytesRef br = dvs.binaryValue();
    boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
    assertEquals(collatorAccepts, luceneAccepts);
  }
}
 

private void verifyResults(Directory indexDir, Directory taxoDir) throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
  IndexSearcher searcher = newSearcher(indexReader);
  
  FacetsCollector collector = new FacetsCollector();
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, collector);

  // tag facets
  Facets tagFacets = new FastTaxonomyFacetCounts("$tags", taxoReader, facetConfig, collector);
  FacetResult result = tagFacets.getTopChildren(10, "tag");
  for (LabelAndValue lv: result.labelValues) {
    if (VERBOSE) {
      System.out.println(lv);
    }
    assertEquals(NUM_DOCS, lv.value.intValue());
  }
  
  // id facets
  Facets idFacets = new FastTaxonomyFacetCounts(taxoReader, facetConfig, collector);
  FacetResult idResult = idFacets.getTopChildren(10, "id");
  assertEquals(NUM_DOCS, idResult.childCount);
  assertEquals(NUM_DOCS * 2, idResult.value); // each "id" appears twice
  
  BinaryDocValues bdv = MultiDocValues.getBinaryValues(indexReader, "bdv");
  BinaryDocValues cbdv = MultiDocValues.getBinaryValues(indexReader, "cbdv");
  for (int i = 0; i < indexReader.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(i, cbdv.nextDoc());
    assertEquals(Integer.parseInt(cbdv.binaryValue().utf8ToString()), Integer.parseInt(bdv.binaryValue().utf8ToString())*2);
  }
  IOUtils.close(indexReader, taxoReader);
}
 

/** Does all the "real work" of tallying up the counts. */
private final void countAll() throws IOException, InterruptedException {
  //System.out.println("ssdv count");

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  List<Future<Void>> results = new ArrayList<>();

  for (LeafReaderContext context : state.getReader().leaves()) {
    results.add(exec.submit(new CountOneSegment(context.reader(), null, ordinalMap, context.ord)));
  }

  for (Future<Void> result : results) {
    try {
      result.get();
    } catch (ExecutionException ee) {
      // Theoretically cause can be null; guard against that.
      Throwable cause = ee.getCause();
      throw IOUtils.rethrowAlways(cause != null ? cause : ee);
    }
  }
}
 

/** Does all the "real work" of tallying up the counts. */
private final void count(List<MatchingDocs> matchingDocs) throws IOException, InterruptedException {

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues && matchingDocs.size() > 1) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  IndexReader reader = state.getReader();
  List<Future<Void>> results = new ArrayList<>();

  for (MatchingDocs hits : matchingDocs) {
    // LUCENE-5090: make sure the provided reader context "matches"
    // the top-level reader passed to the
    // SortedSetDocValuesReaderState, else cryptic
    // AIOOBE can happen:
    if (ReaderUtil.getTopLevelContext(hits.context).reader() != reader) {
      throw new IllegalStateException("the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader");
    }
    
    results.add(exec.submit(new CountOneSegment(hits.context.reader(), hits, ordinalMap, hits.context.ord)));
  }

  for (Future<Void> result : results) {
    try {
      result.get();
    } catch (ExecutionException ee) {
      // Theoretically cause can be null; guard against that.
      Throwable cause = ee.getCause();
      throw IOUtils.rethrowAlways(cause != null ? cause : ee);
    }
  }
}
 

/** Does all the "real work" of tallying up the counts. */
private final void count(List<MatchingDocs> matchingDocs) throws IOException {
  //System.out.println("ssdv count");

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues && matchingDocs.size() > 1) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  IndexReader reader = state.getReader();

  for(MatchingDocs hits : matchingDocs) {

    // LUCENE-5090: make sure the provided reader context "matches"
    // the top-level reader passed to the
    // SortedSetDocValuesReaderState, else cryptic
    // AIOOBE can happen:
    if (ReaderUtil.getTopLevelContext(hits.context).reader() != reader) {
      throw new IllegalStateException("the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader");
    }

    countOneSegment(ordinalMap, hits.context.reader(), hits.context.ord, hits);
  }
}
 

/**
 * Creates an iterator over term, weight and payload fields from the lucene
 * index. setting <code>withPayload</code> to false, implies an iterator
 * over only term and weight.
 */
public DocumentInputIterator(boolean hasPayloads, boolean hasContexts) throws IOException {
  this.hasPayloads = hasPayloads;
  this.hasContexts = hasContexts;
  docCount = reader.maxDoc() - 1;
  weightValues = (weightField != null) ? MultiDocValues.getNumericValues(reader, weightField) : null;
  liveDocs = (reader.leaves().size() > 0) ? MultiBits.getLiveDocs(reader) : null;
  relevantFields = getRelevantFields(new String [] {field, weightField, payloadField, contextsField});
}
 

/** test random generated lines */
protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // line
    Object queryLine = randomQueryLine(shapes);
    Component2D queryLine2D = toLine2D(queryLine);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), POINT_LINE_RELATIONS);
    Query query = newLineQuery(FIELD_NAME, queryRelation, queryLine);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryLine2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryLine);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated polygons */
protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryPolygon = randomQueryPolygon();
    Component2D queryPoly2D = toPolygon2D(queryPolygon);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newPolygonQuery(FIELD_NAME, queryRelation, queryPolygon);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryPolygon);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated point queries */
protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter+1) + " of " + iters + " s=" + s);
    }

    Object[] queryPoints = nextPoints();
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Component2D queryPoly2D;
    Query query;
    if (queryRelation == QueryRelation.CONTAINS) {
      queryPoly2D = toPoint2D(queryPoints[0]);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints[0]);
    } else {
      queryPoly2D = toPoint2D(queryPoints);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints);
    }

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;

      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  rect=Points(" + Arrays.toString(queryPoints) + ")\n");
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated circles */
protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryCircle = randomQueryCircle();
    Component2D queryCircle2D = toCircle2D(queryCircle);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newDistanceQuery(FIELD_NAME, queryRelation, queryCircle);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryCircle2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  distanceQuery=" + queryCircle.toString());
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

public OrdScoreCollector(int maxDoc,
                         int segments,
                         DocValuesProducer collapseValuesProducer,
                         int nullPolicy,
                         IntIntHashMap boostDocsMap,
                         IndexSearcher searcher) throws IOException {
  this.maxDoc = maxDoc;
  this.contexts = new LeafReaderContext[segments];
  List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
  for(int i=0; i<con.size(); i++) {
    contexts[i] = con.get(i);
  }

  this.collapsedSet = new FixedBitSet(maxDoc);
  this.collapseValuesProducer = collapseValuesProducer;
  this.collapseValues = collapseValuesProducer.getSorted(null);

  int valueCount = collapseValues.getValueCount();
  if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
  this.ords = new IntIntDynamicMap(valueCount, -1);
  this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
  this.nullPolicy = nullPolicy;
  if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
    nullScores = new FloatArrayList();
  }

  if(boostDocsMap != null) {
    this.boosts = true;
    this.boostOrds = new IntArrayList();
    this.boostDocs = new IntArrayList();
    int[] bd = new int[boostDocsMap.size()];
    Iterator<IntIntCursor> it =  boostDocsMap.iterator();
    int index = -1;
    while(it.hasNext()) {
      IntIntCursor cursor = it.next();
      bd[++index] = cursor.key;
    }

    Arrays.sort(bd);
    this.mergeBoost = new MergeBoost(bd);
  }
}
 

public void finish() throws IOException {
  if(contexts.length == 0) {
    return;
  }

  int currentContext = 0;
  int currentDocBase = 0;

  this.collapseValues = collapseValuesProducer.getSorted(null);
  if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
  if(ordinalMap != null) {
    this.segmentValues = this.multiSortedDocValues.values[currentContext];
    this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
  } else {
    this.segmentValues = collapseValues;
  }

  int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
  leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
  ScoreAndDoc dummy = new ScoreAndDoc();
  leafDelegate.setScorer(dummy);
  DocIdSetIterator it = new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here
  int globalDoc = -1;
  int nullScoreIndex = 0;
  IntFloatDynamicMap scores = collapseStrategy.getScores();
  FloatArrayList nullScores = collapseStrategy.getNullScores();
  float nullScore = collapseStrategy.getNullScore();

  MergeBoost mergeBoost = collapseStrategy.getMergeBoost();
  while((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {


    while(globalDoc >= nextDocBase) {
      currentContext++;
      currentDocBase = contexts[currentContext].docBase;
      nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
      leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
      leafDelegate.setScorer(dummy);
      if(ordinalMap != null) {
        this.segmentValues = this.multiSortedDocValues.values[currentContext];
        this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
      }
    }

    int contextDoc = globalDoc-currentDocBase;

    if(this.needsScores){
      int ord = -1;
      if(this.ordinalMap != null) {
        //Handle ordinalMapping case
        if (segmentValues.advanceExact(contextDoc)) {
          ord = (int) segmentOrdinalMap.get(segmentValues.ordValue());
        }
      } else {
        //Handle top Level FieldCache or Single Segment Case
        if (segmentValues.advanceExact(globalDoc)) {
          ord = segmentValues.ordValue();
        }
      }

      if(ord > -1) {
        dummy.score = scores.get(ord);
      } else if (mergeBoost != null && mergeBoost.boost(globalDoc)) {
        //It's an elevated doc so no score is needed
        dummy.score = 0F;
      } else if (nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
        dummy.score = nullScore;
      } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
        dummy.score = nullScores.get(nullScoreIndex++);
      }
    }

    dummy.docId = contextDoc;
    leafDelegate.collect(contextDoc);
  }

  if(delegate instanceof DelegatingCollector) {
    ((DelegatingCollector) delegate).finish();
  }
}
 

@Override
public void setUp() throws Exception {
  super.setUp();

  // populate an index with documents - artist, song and weeksAtNumberOne
  dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();

  Field yearField = newTextField("year", "", Field.Store.NO);
  SortedDocValuesField artistField = new SortedDocValuesField("artist",
      new BytesRef(""));
  Field weeksAtNumberOneField = new FloatDocValuesField("weeksAtNumberOne",
      0.0F);
  Field weeksStoredField = new StoredField("weeks", 0.0F);
  Field idField = newStringField("id", "", Field.Store.YES);
  Field songField = newTextField("song", "", Field.Store.NO);
  Field storedArtistField = newTextField("artistName", "", Field.Store.NO);

  doc.add(idField);
  doc.add(weeksAtNumberOneField);
  doc.add(storedArtistField);
  doc.add(songField);
  doc.add(weeksStoredField);
  doc.add(yearField);
  doc.add(artistField);

  parsedRecords.clear();
  for (int i = 0; i < hitsOfThe60s.length; i++) {
    String cols[] = hitsOfThe60s[i].split("\t");
    Record record = new Record(String.valueOf(i), cols[0], cols[1], cols[2],
        Float.parseFloat(cols[3]));
    parsedRecords.put(record.id, record);
    idField.setStringValue(record.id);
    yearField.setStringValue(record.year);
    storedArtistField.setStringValue(record.artist);
    artistField.setBytesValue(new BytesRef(record.artist));
    songField.setStringValue(record.song);
    weeksStoredField.setFloatValue(record.weeks);
    weeksAtNumberOneField.setFloatValue(record.weeks);
    writer.addDocument(doc);
    if (i % 10 == 0) {
      // Causes the creation of multiple segments for our test
      writer.commit();
    }
  }
  reader = writer.getReader();
  writer.close();
  searcher = newSearcher(reader);
  artistDocValues = MultiDocValues.getSortedValues(reader, "artist");
}