org.apache.lucene.index.SortedDocValues Java Examples

public static void visitOrds(SortedDocValues singleDv, DocIdSetIterator disi, OrdFunc ordFunc) throws IOException {
  int doc;
  if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter) {
    FieldCacheImpl.SortedDocValuesImpl.Iter fc = (FieldCacheImpl.SortedDocValuesImpl.Iter) singleDv;
    while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
      ordFunc.handleOrd(doc, fc.getOrd(doc));
    }
  } else {
    while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
      if (singleDv.advanceExact(doc)) {
        ordFunc.handleOrd(doc, singleDv.ordValue());
      } else {
        // TODO: optionally pass in missingOrd?
      }
    }
  }
}
 

/** Converts {@link SortedDocValues} into an {@code Iterable<BytesRef>} for all the values.
 *
 * @deprecated Consume {@link SortedDocValues} instead. */
@Deprecated
public static Iterable<BytesRef> valuesIterable(final SortedDocValues values) {
  return new Iterable<BytesRef>() {
    @Override
    public Iterator<BytesRef> iterator() {
      return new Iterator<BytesRef>() {
        private int nextOrd;
  
        @Override
        public boolean hasNext() {
          return nextOrd < values.getValueCount();
        }

        @Override
        public BytesRef next() {
          try {
            return values.lookupOrd(nextOrd++);
          } catch (IOException e) {
            throw new RuntimeException(e);
          }
        }
      };
    }
  };
}
 

private void accumIntervalsSingle(SortedDocValues sdv, DocIdSetIterator disi, Bits bits) throws IOException {
  // First update the ordinals in the intervals to this segment
  for (FacetInterval interval : intervals) {
    interval.updateContext(sdv);
  }
  int doc;
  while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    if (bits != null && bits.get(doc) == false) {
      continue;
    }
    if (doc > sdv.docID()) {
      sdv.advance(doc);
    }
    if (doc == sdv.docID()) {
      accumInterval(sdv.ordValue());
    }
  }
}
 

/** Wraps a multi-valued SortedSetDocValues as a single-valued view, using the specified selector */
public static SortedDocValues wrap(SortedSetDocValues sortedSet, Type selector) {
  if (sortedSet.getValueCount() >= Integer.MAX_VALUE) {
    throw new UnsupportedOperationException("fields containing more than " + (Integer.MAX_VALUE-1) + " unique terms are unsupported");
  }
  
  SortedDocValues singleton = DocValues.unwrapSingleton(sortedSet);
  if (singleton != null) {
    // it's actually single-valued in practice, but indexed as multi-valued,
    // so just sort on the underlying single-valued dv directly.
    // regardless of selector type, this optimization is safe!
    return singleton;
  } else {
    switch(selector) {
      case MIN: return new MinValue(sortedSet);
      case MAX: return new MaxValue(sortedSet);
      case MIDDLE_MIN: return new MiddleMinValue(sortedSet);
      case MIDDLE_MAX: return new MiddleMaxValue(sortedSet);
      default: 
        throw new AssertionError();
    }
  }
}
 

/** "typical" single-valued faceting: not too many unique values, no prefixing. maps to global ordinals as a separate step */
static void accumSingleSeg(int counts[], SortedDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
  // First count in seg-ord space:
  final int segCounts[];
  if (map == null) {
    segCounts = counts;
  } else {
    segCounts = new int[1+si.getValueCount()];
  }
  
  int doc;
  while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    if (si.advanceExact(doc)) {
      segCounts[1+si.ordValue()]++;
    } else {
      segCounts[0]++;
    }
  }
  
  // migrate to global ords (if necessary)
  if (map != null) {
    migrateGlobal(counts, segCounts, subIndex, map);
  }
}
 

/** accumulates per-segment single-valued facet counts, mapping to global ordinal space on-the-fly */
static void accumSingleGeneric(int counts[], int startTermIndex, SortedDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
  final LongValues ordmap = map == null ? null : map.getGlobalOrds(subIndex);
  int doc;
  while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    int term;
    if (si.advanceExact(doc)) {
      term = si.ordValue();
    } else {
      term = -1;
    }
    if (map != null && term >= 0) {
      term = (int) ordmap.get(term);
    }
    int arrIdx = term-startTermIndex;
    if (arrIdx>=0 && arrIdx<counts.length) counts[arrIdx]++;
  }
}
 

public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  groups = new LongObjectHashMap<>(numGroups);
  DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
  int group;
  while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    groups.put(group, getCollector(limit, sort));
  }

  this.collapsedSet = collapsedSet;
  this.groupBits = groupBits;
  this.docValues = docValues;
  if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
}
 

public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 

@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }

  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }
  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (foundOrds.get(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = approximationWeight.scorer(context);
  if (approximationScorer == null) {
    return null;
  }
  if (globalOrds != null) {
    return new OrdinalMapScorer(this, score(), foundOrds, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } {
    return new SegmentOrdinalScorer(this, score(), foundOrds, values, approximationScorer.iterator());
  }
}
 

@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
  SortedDocValues sortedDocValues = _sortedDocValuesCache.get(field.number);
  if (sortedDocValues != null) {
    return sortedDocValues;
  }
  synchronized (_sortedDocValuesCache) {
    sortedDocValues = _sortedDocValuesCache.get(field.number);
    if (sortedDocValues != null) {
      return sortedDocValues;
    }
    sortedDocValues = newSortedDocValues(field);
    if (_cache && sortedDocValues != null) {
      _sortedDocValuesCache.put(field.number, sortedDocValues);
    }
    return sortedDocValues;
  }
}
 

public OrdLongStrategy(int maxDoc,
                       int nullPolicy,
                       int valueCount,
                       GroupHeadSelector groupHeadSelector,
                       boolean needsScores,
                       IntIntHashMap boostDocs, SortedDocValues values) throws IOException {
  super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
  this.field = groupHeadSelector.selectorText;

  assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);

  if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
    comp = new MaxLongComp();
    this.ordVals = new IntLongDynamicMap(valueCount, Long.MIN_VALUE);
  } else {
    this.nullVal = Long.MAX_VALUE;
    comp = new MinLongComp();
    this.ordVals = new IntLongDynamicMap(valueCount, Long.MAX_VALUE);
  }
}
 

public OrdIntStrategy(int maxDoc,
                      int nullPolicy,
                      int valueCount,
                      GroupHeadSelector groupHeadSelector,
                      boolean needsScores,
                      IntIntHashMap boostDocs,
                      SortedDocValues values) throws IOException {
  super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
  this.field = groupHeadSelector.selectorText;

  assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);

  if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
    comp = new MaxIntComp();
    this.ordVals = new IntIntDynamicMap(valueCount, Integer.MIN_VALUE);
  } else {
    comp = new MinIntComp();
    this.ordVals = new IntIntDynamicMap(valueCount, Integer.MAX_VALUE);
    this.nullVal = Integer.MAX_VALUE;
  }
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = in.scorer(context);
  if (approximationScorer == null) {
    return null;
  } else if (globalOrds != null) {
    return new OrdinalMapScorer(this, collector, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } else {
    return new SegmentOrdinalScorer(this, collector, values, approximationScorer.iterator());
  }
}
 

public OrdFloatStrategy(int maxDoc,
                        int nullPolicy,
                        int valueCount,
                        GroupHeadSelector groupHeadSelector,
                        boolean needsScores,
                        IntIntHashMap boostDocs,
                        SortedDocValues values) throws IOException {
  super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
  this.field = groupHeadSelector.selectorText;

  assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);

  if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
    comp = new MaxFloatComp();
    this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
    this.nullVal = -Float.MAX_VALUE;
  } else {
    comp = new MinFloatComp();
    this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE);
    this.nullVal = Float.MAX_VALUE;
  }
}
 

private FieldComparator<?> getStringComparator(int numHits) {
  return new FieldComparator.TermOrdValComparator(numHits, getField(), missingValue == STRING_LAST) {

    @Override
    protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
      SortedSetDocValues sortedSet = DocValues.getSortedSet(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptySorted();
      }
      return BlockJoinSelector.wrap(sortedSet, type, parents, toIter(children));
    }

  };
}
 

@Override
public SortedDocValues getOrdinalsValues(String type) {
    AtomicOrdinalsFieldData atomicFieldData = typeToIds.get(type);
    if (atomicFieldData != null) {
        return MultiValueMode.MIN.select(atomicFieldData.getOrdinalsValues());
    } else {
        return DocValues.emptySorted();
    }
}
 

@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
  SortedDocValues values = in.getSortedDocValues(field);
  if (values != null) {
    return values;
  }
  Type v = getType(field);
  if (v == Type.SORTED) {
    return FieldCache.DEFAULT.getTermsIndex(in, field);
  } else {
    return null;
  }
}
 

/** accumulates per-segment single-valued facet counts */
static void accumSingle(int counts[], int startTermIndex, SortedDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
  if (startTermIndex == -1 && (map == null || si.getValueCount() < disi.cost()*10)) {
    // no prefixing, not too many unique values wrt matching docs (lucene/facets heuristic): 
    //   collect separately per-segment, then map to global ords
    accumSingleSeg(counts, si, disi, subIndex, map);
  } else {
    // otherwise: do collect+map on the fly
    accumSingleGeneric(counts, startTermIndex, si, disi, subIndex, map);
  }
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
    DocIdSet parentsSet = parentFilter.getDocIdSet(context, null);
    if (Lucene.isEmpty(parentsSet) || remaining == 0) {
        return null;
    }

    // We can't be sure of the fact that liveDocs have been applied, so we apply it here. The "remaining"
    // count down (short circuit) logic will then work as expected.
    DocIdSetIterator parents = BitsFilteredDocIdSet.wrap(parentsSet, context.reader().getLiveDocs()).iterator();

    if (parents != null) {
        SortedDocValues bytesValues = collector.globalIfd.load(context).getOrdinalsValues(parentType);
        if (bytesValues == null) {
            return null;
        }

        if (minChildren > 0 || maxChildren != 0 || scoreType == ScoreType.NONE) {
            switch (scoreType) {
            case NONE:
                DocIdSetIterator parentIdIterator = new CountParentOrdIterator(this, parents, collector, bytesValues,
                        minChildren, maxChildren);
                return ConstantScorer.create(parentIdIterator, this, queryWeight);
            case AVG:
                return new AvgParentCountScorer(this, parents, collector, bytesValues, minChildren, maxChildren);
            default:
                return new ParentCountScorer(this, parents, collector, bytesValues, minChildren, maxChildren);
            }
        }
        switch (scoreType) {
        case AVG:
            return new AvgParentScorer(this, parents, collector, bytesValues);
        default:
            return new ParentScorer(this, parents, collector, bytesValues);
        }
    }
    return null;
}
 

/** Looks up the nest path.  If there is none, returns {@link #ANON_CHILD_KEY}. */
private String getPathByDocId(int segDocId, SortedDocValues segPathDocValues) throws IOException {
  if (!isNestedSchema) {
    return ANON_CHILD_KEY;
  }
  int numToAdvance = segPathDocValues.docID() == -1 ? segDocId : segDocId - (segPathDocValues.docID());
  assert numToAdvance >= 0;
  boolean advanced = segPathDocValues.advanceExact(segDocId);
  return advanced ? segPathDocValues.binaryValue().utf8ToString(): "";
}
 

public static SortedDocValues getSortedDocValues(QueryContext context, SchemaField field, QParser qparser) throws IOException {
  SortedDocValues si = context.searcher().getSlowAtomicReader().getSortedDocValues( field.getName() );
  // if (!field.hasDocValues() && (field.getType() instanceof StrField || field.getType() instanceof TextField)) {
  // }

  return si == null ? DocValues.emptySorted() : si;
}
 

/** Wraps the provided {@link SortedSetDocValues} in order to only select
 *  one value per parent among its {@code children} using the configured
 *  {@code selection} type. */
public static SortedDocValues wrap(SortedSetDocValues sortedSet, Type selection, BitSet parents, DocIdSetIterator children) {
  SortedDocValues values;
  switch (selection) {
    case MIN:
      values = SortedSetSelector.wrap(sortedSet, SortedSetSelector.Type.MIN);
      break;
    case MAX:
      values = SortedSetSelector.wrap(sortedSet, SortedSetSelector.Type.MAX);
      break;
    default:
      throw new AssertionError();
  }
  return wrap(values, selection, parents, children);
}
 

@Override
public FieldComparator<?> getComparator(int numHits, int sortPos) {
  return new FieldComparator.TermOrdValComparator(numHits, getField(), missingValue == STRING_LAST) {
    @Override
    protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
      return SortedSetSelector.wrap(DocValues.getSortedSet(context.reader(), field), selector);
    }
  };
}
 

/** Wraps the provided {@link SortedDocValues} in order to only select
 *  one value per parent among its {@code children} using the configured
 *  {@code selection} type. */
public static SortedDocValues wrap(final SortedDocValues values, Type selection, BitSet parents, DocIdSetIterator children) {
  if (values.docID() != -1) {
    throw new IllegalArgumentException("values iterator was already consumed: values.docID=" + values.docID());
  }
  return ToParentDocValues.wrap(values, selection, parents, children);
}
 

private SortedDocValues newSortedDocValues(FieldInfo field) throws IOException {
  final int valueCount = (int) binaries.get(field.number).count;
  final BinaryDocValues binary = getBinary(field);
  Tracer trace = Trace.trace("getSorted - BlockPackedReader - create");
  final BlockPackedReader ordinals;
  try {
    NumericEntry entry = ords.get(field.number);
    IndexInput data = this.data.clone();
    data.seek(entry.offset);
    ordinals = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true);
  } finally {
    trace.done();
  }
  return new SortedDocValues() {

    @Override
    public int getOrd(int docID) {
      return (int) ordinals.get(docID);
    }

    @Override
    public void lookupOrd(int ord, BytesRef result) {
      binary.get(ord, result);
    }

    @Override
    public int getValueCount() {
      return valueCount;
    }
  };
}
 

public StringValue(SortedDocValues globalDocValues, String field, IntComp comp)  {
  this.globalDocValues = globalDocValues;
  this.docValues = globalDocValues;
  if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) globalDocValues).mapping;
  }
  this.field = field;
  this.comp = comp;
  this.currentOrd = comp.resetValue();
  this.present = false;
}
 

private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception { 
  SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");
  for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {
    Document doc = is.doc(docID);
    String s = doc.getField("field").stringValue();
    boolean collatorAccepts = collate(collator, s, startPoint) >= 0 && collate(collator, s, endPoint) <= 0;
    assertEquals(docID, dvs.nextDoc());
    BytesRef br = dvs.binaryValue();
    boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
    assertEquals(startPoint + " <= " + s + " <= " + endPoint, collatorAccepts, luceneAccepts);
  }
}
 

private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception { 
  SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");
  for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {
    Document doc = is.doc(docID);
    String s = doc.getField("field").stringValue();
    boolean collatorAccepts = collator.compare(s, startPoint) >= 0 && collator.compare(s, endPoint) <= 0;
    assertEquals(docID, dvs.nextDoc());
    BytesRef br = dvs.binaryValue();
    boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
    assertEquals(collatorAccepts, luceneAccepts);
  }
}
 

public void testSortedSelector() throws IOException {
  final BitSet parents = new FixedBitSet(20);
  parents.set(0);
  parents.set(5);
  parents.set(6);
  parents.set(10);
  parents.set(15);
  parents.set(19);

  final BitSet children = new FixedBitSet(20);
  children.set(2);
  children.set(3);
  children.set(4);
  children.set(12);
  children.set(17);

  final int[] ords = new int[20];
  Arrays.fill(ords, -1);
  ords[2] = 5;
  ords[3] = 7;
  ords[4] = 3;
  ords[12] = 10;
  ords[18] = 10;

  final SortedDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MIN, parents, toIter(children));
  assertEquals(5, nextDoc(mins,5));
  assertEquals(3, mins.ordValue());
  assertEquals(15, nextDoc(mins,15));
  assertEquals(10, mins.ordValue());
  assertNoMoreDoc(mins, 20);

  final SortedDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MAX, parents, toIter(children));
  assertEquals(5, nextDoc(maxs,5));
  assertEquals(7, maxs.ordValue());
  assertEquals(15, nextDoc(maxs,15));
  assertEquals(10, maxs.ordValue());
  assertNoMoreDoc( maxs,20);
}