Java Code Examples for org.apache.lucene.index.DocValues#getSorted()

public void facetMissingNum(int docID) throws IOException {
  if (topLevelSortedValues == null) {
    topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
  }
  
  if (docID > topLevelSortedValues.docID()) {
    topLevelSortedValues.advance(docID);
  }
 
  if (docID == topLevelSortedValues.docID()) {
    int ord = topLevelSortedValues.ordValue();
    Integer missingCount = missingStats.get(ord);
    if (missingCount == null) {
      missingStats.put(ord, 1);
    } else {
      missingStats.put(ord, missingCount + 1);
    }
  }
}
 

@Override
public void doSetNextReader(LeafReaderContext context) throws IOException {
  docValues = DocValues.getSorted(context.reader(), fieldName);

  // figure out what ord maps to true
  int numOrds = docValues.getValueCount();
  // if no values in the segment, default trueOrd to something other then -1 (missing)
  int trueOrd = -2;
  for (int i=0; i<numOrds; i++) {
    final BytesRef br = docValues.lookupOrd(i);
    if (br.length==1 && br.bytes[br.offset]=='T') {
      trueOrd = i;
      break;
    }
  }

  this.trueOrd = trueOrd;
}
 

@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }

  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }
  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (foundOrds.get(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = approximationWeight.scorer(context);
  if (approximationScorer == null) {
    return null;
  }
  if (globalOrds != null) {
    return new OrdinalMapScorer(this, score(), foundOrds, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } {
    return new SegmentOrdinalScorer(this, score(), foundOrds, values, approximationScorer.iterator());
  }
}
 

/**
 * Returns an {@link IndexSorter} used for sorting index segments by this SortField.
 *
 * If the SortField cannot be used for index sorting (for example, if it uses scores or
 * other query-dependent values) then this method should return {@code null}
 *
 * SortFields that implement this method should also implement a companion
 * {@link SortFieldProvider} to serialize and deserialize the sort in index segment
 * headers
 *
 * @lucene.experimental
 */
public IndexSorter getIndexSorter() {
  switch (type) {
    case STRING:
      return new IndexSorter.StringSorter(Provider.NAME, missingValue, reverse, reader -> DocValues.getSorted(reader, field));
    case INT:
      return new IndexSorter.IntSorter(Provider.NAME, (Integer)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    case LONG:
      return new IndexSorter.LongSorter(Provider.NAME, (Long)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    case DOUBLE:
      return new IndexSorter.DoubleSorter(Provider.NAME, (Double)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    case FLOAT:
      return new IndexSorter.FloatSorter(Provider.NAME, (Float)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    default: return null;
  }
}
 

@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }
  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }

  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (collector.match(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  float score = collector.score(ord);
  return Explanation.match(score, "A match, join value " + Term.toString(joinValue));
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = in.scorer(context);
  if (approximationScorer == null) {
    return null;
  } else if (globalOrds != null) {
    return new OrdinalMapScorer(this, collector, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } else {
    return new SegmentOrdinalScorer(this, collector, values, approximationScorer.iterator());
  }
}
 

@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
  this.docValues = DocValues.getSorted(readerContext.reader(), field);
  this.ordsToGroupIds.clear();
  BytesRef scratch = new BytesRef();
  for (int i = 0; i < values.size(); i++) {
    values.get(i, scratch);
    int ord = this.docValues.lookupTerm(scratch);
    if (ord >= 0)
      ordsToGroupIds.put(ord, i);
  }
}
 

public void setNextReader(LeafReaderContext context) throws IOException {
  if (ordinalMap != null) {
    toGlobal = ordinalMap.getGlobalOrds(context.ord);
  }
  docValues = DocValues.getSorted(context.reader(), field);
  lastDocID = 0;
}
 

@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 

@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 

static SortedDocValues open(LeafReaderContext context, String field) throws IOException {
  try {
    return DocValues.getSorted(context.reader(), field);
  } catch (RuntimeException e) {
    throw new DocTermsIndexException(field, e);
  }
}
 

public void testMinMaxDocs() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(
      random(),
      dir,
      newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))
  );

  int minChildDocsPerParent = 2;
  int maxChildDocsPerParent = 16;
  int numParents = RandomNumbers.randomIntBetween(random(), 16, 64);
  int[] childDocsPerParent = new int[numParents];
  for (int p = 0; p < numParents; p++) {
    String parentId = Integer.toString(p);
    Document parentDoc = new Document();
    parentDoc.add(new StringField("id", parentId, Field.Store.YES));
    parentDoc.add(new StringField("type", "to", Field.Store.NO));
    parentDoc.add(new SortedDocValuesField("join_field", new BytesRef(parentId)));
    iw.addDocument(parentDoc);
    int numChildren = RandomNumbers.randomIntBetween(random(), minChildDocsPerParent, maxChildDocsPerParent);
    childDocsPerParent[p] = numChildren;
    for (int c = 0; c < numChildren; c++) {
      String childId = Integer.toString(p + c);
      Document childDoc = new Document();
      childDoc.add(new StringField("id", childId, Field.Store.YES));
      childDoc.add(new StringField("type", "from", Field.Store.NO));
      childDoc.add(new SortedDocValuesField("join_field", new BytesRef(parentId)));
      iw.addDocument(childDoc);
    }
  }
  iw.close();

  IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(dir));
  SortedDocValues[] values = new SortedDocValues[searcher.getIndexReader().leaves().size()];
  for (LeafReaderContext leadContext : searcher.getIndexReader().leaves()) {
    values[leadContext.ord] = DocValues.getSorted(leadContext.reader(), "join_field");
  }
  OrdinalMap ordinalMap = OrdinalMap.build(
      null, values, PackedInts.DEFAULT
  );
  Query fromQuery = new TermQuery(new Term("type", "from"));
  Query toQuery = new TermQuery(new Term("type", "to"));

  int iters = RandomNumbers.randomIntBetween(random(), 3, 9);
  for (int i = 1; i <= iters; i++) {
    final ScoreMode scoreMode = ScoreMode.values()[random().nextInt(ScoreMode.values().length)];
    int min = RandomNumbers.randomIntBetween(random(), minChildDocsPerParent, maxChildDocsPerParent - 1);
    int max = RandomNumbers.randomIntBetween(random(), min, maxChildDocsPerParent);
    if (VERBOSE) {
      System.out.println("iter=" + i);
      System.out.println("scoreMode=" + scoreMode);
      System.out.println("min=" + min);
      System.out.println("max=" + max);
    }
    Query joinQuery = JoinUtil.createJoinQuery("join_field", fromQuery, toQuery, searcher, scoreMode, ordinalMap, min, max);
    TotalHitCountCollector collector = new TotalHitCountCollector();
    searcher.search(joinQuery, collector);
    int expectedCount = 0;
    for (int numChildDocs : childDocsPerParent) {
      if (numChildDocs >= min && numChildDocs <= max) {
        expectedCount++;
      }
    }
    assertEquals(expectedCount, collector.getTotalHits());
  }

  searcher.getIndexReader().close();
  dir.close();
}
 

/** Retrieves the SortedDocValues for the field in this segment */
protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
  return DocValues.getSorted(context.reader(), field);
}
 

@Override
public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
  return DocValues.getSorted(searcher.getSlowAtomicReader(), collapseField);
}
 

@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
  super.setNextReader(readerContext);
  values = DocValues.getSorted(readerContext.reader(), sf.getName());
}
 

public boolean facetTermNum(int docID, int statsTermNum) throws IOException {
  if (topLevelSortedValues == null) {
    topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
  }

  if (docID > topLevelSortedValues.docID()) {
    topLevelSortedValues.advance(docID);
  }
 
  int term;
  if (docID == topLevelSortedValues.docID()) {
    term = topLevelSortedValues.ordValue();
  } else {
    term = -1;
  }
  
  int arrIdx = term;
  if (arrIdx >= 0 && arrIdx < topLevelSortedValues.getValueCount()) {
    final String key;
    if (term == -1) {
      key = null;
    } else {
      key = topLevelSortedValues.lookupOrd(term).utf8ToString();
    }
    while (facetStatsTerms.size() <= statsTermNum) {
      facetStatsTerms.add(new HashMap<String, Integer>());
    }
    
    
    final Map<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
    Integer statsTermCount = statsTermCounts.get(key);
    if (statsTermCount == null) {
      statsTermCounts.put(key, 1);
    } else {
      statsTermCounts.put(key, statsTermCount + 1);
    }
    return true;
  }
  
  return false;
}
 

@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
  if (segmentFacetCounts != null) {
    segmentResults.add(createSegmentResult());
  }

  groupFieldTermsIndex = DocValues.getSorted(context.reader(), groupField);
  facetFieldDocTermOrds = DocValues.getSortedSet(context.reader(), facetField);
  facetFieldNumTerms = (int) facetFieldDocTermOrds.getValueCount();
  if (facetFieldNumTerms == 0) {
    facetOrdTermsEnum = null;
  } else {
    facetOrdTermsEnum = facetFieldDocTermOrds.termsEnum();
  }
  // [facetFieldNumTerms() + 1] for all possible facet values and docs not containing facet field
  segmentFacetCounts = new int[facetFieldNumTerms + 1];
  segmentTotalCount = 0;

  segmentGroupedFacetHits.clear();
  for (GroupedFacetHit groupedFacetHit : groupedFacetHits) {
    int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.lookupTerm(groupedFacetHit.groupValue);
    if (groupedFacetHit.groupValue != null && groupOrd < 0) {
      continue;
    }

    int facetOrd;
    if (groupedFacetHit.facetValue != null) {
      if (facetOrdTermsEnum == null || !facetOrdTermsEnum.seekExact(groupedFacetHit.facetValue)) {
        continue;
      }
      facetOrd = (int) facetOrdTermsEnum.ord();
    } else {
      facetOrd = facetFieldNumTerms;
    }

    // (facetFieldDocTermOrds.numTerms() + 1) for all possible facet values and docs not containing facet field
    int segmentGroupedFacetsIndex = groupOrd * (facetFieldNumTerms + 1) + facetOrd;
    segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
  }

  if (facetPrefix != null) {
    TermsEnum.SeekStatus seekStatus;
    if (facetOrdTermsEnum != null) {
      seekStatus = facetOrdTermsEnum.seekCeil(facetPrefix);
    } else {
      seekStatus = TermsEnum.SeekStatus.END;
    }

    if (seekStatus != TermsEnum.SeekStatus.END) {
      startFacetOrd = (int) facetOrdTermsEnum.ord();
    } else {
      startFacetOrd = 0;
      endFacetOrd = 0;
      return;
    }

    BytesRefBuilder facetEndPrefix = new BytesRefBuilder();
    facetEndPrefix.append(facetPrefix);
    facetEndPrefix.append(UnicodeUtil.BIG_TERM);
    seekStatus = facetOrdTermsEnum.seekCeil(facetEndPrefix.get());
    if (seekStatus != TermsEnum.SeekStatus.END) {
      endFacetOrd = (int) facetOrdTermsEnum.ord();
    } else {
      endFacetOrd = facetFieldNumTerms; // Don't include null...
    }
  } else {
    startFacetOrd = 0;
    endFacetOrd = facetFieldNumTerms + 1;
  }
}