org.apache.lucene.index.DocValues Java Examples

public void facetMissingNum(int docID) throws IOException {
  if (topLevelSortedValues == null) {
    topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
  }
  
  if (docID > topLevelSortedValues.docID()) {
    topLevelSortedValues.advance(docID);
  }
 
  if (docID == topLevelSortedValues.docID()) {
    int ord = topLevelSortedValues.ordValue();
    Integer missingCount = missingStats.get(ord);
    if (missingCount == null) {
      missingStats.put(ord, 1);
    } else {
      missingStats.put(ord, missingCount + 1);
    }
  }
}
 

public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 

@Override
public FunctionValues getValues(Map<Object, Object> context, LeafReaderContext readerContext) throws IOException {
  SortedSetDocValues sortedSet = DocValues.getSortedSet(readerContext.reader(), field);
  SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
  return new DocTermsIndexDocValues(this, view) {
    @Override
    protected String toTerm(String readableValue) {
      return readableValue;
    }

    @Override
    public Object objectVal(int doc) throws IOException {
      return strVal(doc);
    }
  };
}
 

public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
  int val;
  SortValue sortValue = sortDoc.getSortValue(this.field);
  if (sortValue != null) {
    if (sortValue.isPresent()) {
      val = (int) sortValue.getCurrentValue();
    } else { //empty-value
      return false;
    }
  } else {
    // field is not part of 'sort' param, but part of 'fl' param
    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
    if (vals.advance(sortDoc.docId) == sortDoc.docId) {
      val = (int) vals.longValue();
    } else {
      return false;
    }
  }
  ew.put(this.field, val);
  return true;
}
 

@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
  final int maxDoc = context.reader().maxDoc();
  FixedBitSet bits = new FixedBitSet(maxDoc);
  final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
  int docID;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    final int count = values.docValueCount();
    for (int i = 0; i < count; ++i) {
      final long v = values.nextValue();
      if (v >= min && v <= max) {
        bits.set(docID);
        break;
      }
    }
  }
  return bits;
}
 

@Benchmark
public LongObjectHashMap<Long> measureGroupingOnNumericDocValues() throws Exception {
    Weight weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    LeafReaderContext leaf = searcher.getTopReaderContext().leaves().get(0);
    Scorer scorer = weight.scorer(leaf);
    NumericDocValues docValues = DocValues.getNumeric(leaf.reader(), "x");
    DocIdSetIterator docIt = scorer.iterator();
    LongObjectHashMap<Long> sumByKey = new LongObjectHashMap<>();
    for (int docId = docIt.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docIt.nextDoc()) {
        if (docValues.advanceExact(docId)) {
            long number = docValues.longValue();
            sumByKey.compute(number, (key, oldValue) -> {
                if (oldValue == null) {
                    return number;
                } else {
                    return oldValue + number;
                }
            });
        }
    }
    return sumByKey;
}
 

private void countAllMultiValued(IndexReader reader, String field) throws IOException {

    for (LeafReaderContext context : reader.leaves()) {

      SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        // this field has no doc values for this segment
        continue;
      }
      NumericDocValues singleValues = DocValues.unwrapSingleton(values);
      if (singleValues != null) {
        countAllOneSegment(singleValues);
      } else {
        int doc;
        while ((doc = values.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
          int limit = values.docValueCount();
          totCount += limit;
          for (int i = 0; i < limit; i++) {
            increment(values.nextValue());
          }
        }
      }
    }
  }
 

@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }

  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }
  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (foundOrds.get(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
 

public static AtomicOrdinalsFieldData empty() {
    return new AbstractAtomicOrdinalsFieldData() {

        @Override
        public long ramBytesUsed() {
            return 0;
        }
        
        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

        @Override
        public void close() {
        }

        @Override
        public RandomAccessOrds getOrdinalsValues() {
            return DocValues.emptySortedSet();
        }
    };
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = in.scorer(context);
  if (approximationScorer == null) {
    return null;
  } else if (globalOrds != null) {
    return new OrdinalMapScorer(this, collector, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } else {
    return new SegmentOrdinalScorer(this, collector, values, approximationScorer.iterator());
  }
}
 

private FieldComparator<?> getIntComparator(int numHits) {
  return new FieldComparator.IntComparator(numHits, getField(), (Integer) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
    }
  };
}
 

private FieldComparator<?> getFloatComparator(int numHits) {
  return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableFloatBits((int) super.longValue());
        }
      };
    }
  };
}
 

private SortedSetDocValues validateAndFetchDocValues(SolrIndexSearcher solrSearcher, String fieldName, String querySide) throws IOException {
  final IndexSchema schema = solrSearcher.getSchema();
  final SchemaField field = schema.getFieldOrNull(fieldName);
  if (field == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, querySide + " field '" + fieldName + "' does not exist");
  }

  if (!field.hasDocValues()) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        "'top-level' join queries require both 'from' and 'to' fields to have docValues, but " + querySide +
            " field [" + fieldName +  "] does not.");
  }

  final LeafReader leafReader = solrSearcher.getSlowAtomicReader();
  if (field.multiValued()) {
    return DocValues.getSortedSet(leafReader, fieldName);
  }
  return DocValues.singleton(DocValues.getSorted(leafReader, fieldName));
}
 

@Benchmark
public LongObjectHashMap<Long> measureGroupingOnSortedNumericDocValues() throws Exception {
    var weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    var leaf = searcher.getTopReaderContext().leaves().get(0);
    var scorer = weight.scorer(leaf);
    var docValues = DocValues.getSortedNumeric(leaf.reader(), "y");
    var docIt = scorer.iterator();
    LongObjectHashMap<Long> sumByKey = new LongObjectHashMap<>();
    for (int docId = docIt.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docIt.nextDoc()) {
        if (docValues.advanceExact(docId)) {
            if (docValues.docValueCount() == 1) {
                long number = docValues.nextValue();
                sumByKey.compute(number, (key, oldValue) -> {
                    if (oldValue == null) {
                        return number;
                    } else {
                        return oldValue + number;
                    }
                });
            }
        }
    }
    return sumByKey;
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      Random random = new Random(context.docBase ^ seed);
      final int maxDoc = context.reader().maxDoc();
      final NumericDocValues idSource = DocValues.getNumeric(context.reader(), "id");
      assertNotNull(idSource);
      final FixedBitSet bits = new FixedBitSet(maxDoc);
      for(int docID=0;docID<maxDoc;docID++) {
        assertEquals(docID, idSource.nextDoc());
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
          matchValues.add(docValues.get((int) idSource.longValue()));
        }
      }

      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }
  };
}
 

@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);

        NumericDocValues single = DocValues.unwrapSingleton(raw);
        if (single != null) {
            return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw));
        } else {
            return new MultiFloatValues(raw);
        }
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 

@Override
public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
  LeafReader reader = context.reader();
  FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info != null) {
    Geo3DDocValuesField.checkCompatible(info);
  }
  currentDocs = DocValues.getSortedNumeric(reader, field);
  return this;
}
 

@Override
public RandomAccessOrds getOrdinalsValues() {
    try {
        return FieldData.maybeSlowRandomAccessOrds(DocValues.getSortedSet(reader, field));
    } catch (IOException e) {
        throw new IllegalStateException("cannot load docvalues", e);
    }
}
 

@Override
public SortedNumericDocValues getLongValues() {
    try {
        return DocValues.getSortedNumeric(reader, field);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 

@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);
        return FieldData.sortableLongBitsToDoubles(raw);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 

/**
 * Returns the FunctionValues used by the function query.
 */
@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
  LeafReader reader = readerContext.reader();

  final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
  final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());

  return DoubleValues.withDefault(new DoubleValues() {

    private final Point from = DistanceValueSource.this.from;
    private final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();

    @Override
    public double doubleValue() throws IOException {
      double x = Double.longBitsToDouble(ptX.longValue());
      double y = Double.longBitsToDouble(ptY.longValue());
      return calculator.distance(from, x, y) * multiplier;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return ptX.advanceExact(doc) && ptY.advanceExact(doc);
    }

  }, nullValue);
}
 

@Override
public RandomAccessOrds ordinals(ValuesHolder values) {
    if (multiValued) {
        return new MultiDocs(this, values);
    } else {
        return (RandomAccessOrds) DocValues.singleton(new SingleDocs(this, values));
    }
}
 

@Override
public BytesBinaryDVAtomicFieldData load(LeafReaderContext context) {
    try {
        return new BytesBinaryDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName()));
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 

static SortedDocValues open(LeafReaderContext context, String field) throws IOException {
  try {
    return DocValues.getSorted(context.reader(), field);
  } catch (RuntimeException e) {
    throw new DocTermsIndexException(field, e);
  }
}
 

@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 

@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  Weight fallbackWeight = fallbackQuery.createWeight(searcher, scoreMode, boost);

  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      SortedNumericDocValues sortedNumericValues = DocValues.getSortedNumeric(context.reader(), field);
      NumericDocValues numericValues = DocValues.unwrapSingleton(sortedNumericValues);

      if (numericValues != null) {
        Sort indexSort = context.reader().getMetaData().getSort();
        if (indexSort != null
            && indexSort.getSort().length > 0
            && indexSort.getSort()[0].getField().equals(field)) {

          SortField sortField = indexSort.getSort()[0];
          DocIdSetIterator disi = getDocIdSetIterator(sortField, context, numericValues);
          return new ConstantScoreScorer(this, score(), scoreMode, disi);
        }
      }
      return fallbackWeight.scorer(context);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      // Both queries should always return the same values, so we can just check
      // if the fallback query is cacheable.
      return fallbackWeight.isCacheable(ctx);
    }
  };
}
 

@Override
public FieldComparator<?> getComparator(int numHits, int sortPos) {
  return new FieldComparator.TermOrdValComparator(numHits, getField(), missingValue == STRING_LAST) {
    @Override
    protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
      return SortedSetSelector.wrap(DocValues.getSortedSet(context.reader(), field), selector);
    }
  };
}
 

public void setNextReader(LeafReaderContext context) throws IOException {
  if (ordinalMap != null) {
    toGlobal = ordinalMap.getGlobalOrds(context.ord);
  }
  docValues = DocValues.getSorted(context.reader(), field);
  lastDocID = 0;
}
 

@Override
public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
  LeafReader reader = context.reader();
  FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info != null) {
    XYDocValuesField.checkCompatible(info);
  }
  currentDocs = DocValues.getSortedNumeric(reader, field);
  valuesDocID = -1;
  return this;
}