org.apache.lucene.index.SortedNumericDocValues Java Examples

/**
 * Return a {@link MurmurHash3Values} instance that returns each value as its hash.
 */
public static MurmurHash3Values cast(final SortedNumericDocValues values) {
    return new MurmurHash3Values() {
        @Override
        public void setDocument(int docId) {
            values.setDocument(docId);
        }
        @Override
        public int count() {
            return values.count();
        }
        @Override
        public long valueAt(int index) {
            return values.valueAt(index);
        }
    };
}
 

private FieldComparator<?> getDoubleComparator(int numHits) {
  return new FieldComparator.DoubleComparator(numHits, getField(), (Double) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableDoubleBits(super.longValue());
        }
      };
    }
  };
}
 

private FieldComparator<?> getFloatComparator(int numHits) {
  return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableFloatBits((int) super.longValue());
        }
      };
    }
  };
}
 

private FieldComparator<?> getLongComparator(int numHits) {
  return new FieldComparator.LongComparator(numHits, getField(), (Long) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
    }
  };
}
 

private FieldComparator<?> getIntComparator(int numHits) {
  return new FieldComparator.IntComparator(numHits, getField(), (Integer) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
    }
  };
}
 

@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
  final int maxDoc = context.reader().maxDoc();
  FixedBitSet bits = new FixedBitSet(maxDoc);
  final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
  int docID;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    final int count = values.docValueCount();
    for (int i = 0; i < count; ++i) {
      final long v = values.nextValue();
      if (v >= min && v <= max) {
        bits.set(docID);
        break;
      }
    }
  }
  return bits;
}
 

private void countAllMultiValued(IndexReader reader, String field) throws IOException {

    for (LeafReaderContext context : reader.leaves()) {

      SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        // this field has no doc values for this segment
        continue;
      }
      NumericDocValues singleValues = DocValues.unwrapSingleton(values);
      if (singleValues != null) {
        countAllOneSegment(singleValues);
      } else {
        int doc;
        while ((doc = values.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
          int limit = values.docValueCount();
          totCount += limit;
          for (int i = 0; i < limit; i++) {
            increment(values.nextValue());
          }
        }
      }
    }
  }
 

private Optional<DocValues> createSortedNumericDocValues(int docid, String field, DocValuesType dvType)
    throws IOException {
  SortedNumericDocValues snvalues = IndexUtils.getSortedNumericDocValues(reader, field);

  if (snvalues.advanceExact(docid)) {
    List<Long> numericValues = new ArrayList<>();

    int dvCount = snvalues.docValueCount();
    for (int i = 0; i < dvCount; i++) {
      numericValues.add(snvalues.nextValue());
    }

    DocValues dv = DocValues.of(
        dvType,
        Collections.emptyList(),
        numericValues
    );
    return Optional.of(dv);
  }

  return Optional.empty();
}
 

@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);
        return FieldData.sortableLongBitsToDoubles(raw);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 

@Override
public SortedNumericDocValues getLongValues() {
    try {
        return DocValues.getSortedNumeric(reader, field);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 

@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);

        NumericDocValues single = DocValues.unwrapSingleton(raw);
        if (single != null) {
            return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw));
        } else {
            return new MultiFloatValues(raw);
        }
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 

public static AtomicNumericFieldData empty(final int maxDoc) {
    return new AtomicLongFieldData(0) {

        @Override
        public SortedNumericDocValues getLongValues() {
            return DocValues.emptySortedNumeric(maxDoc);
        }

        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

    };
}
 

public void testDocValues_resetIterator() throws Exception {
  Document doc = new Document();

  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("f")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c")));

  doc.add(new SortedNumericDocValuesField("sorted_numeric", 33L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 31L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 30L));

  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
  assertEquals(3, sortedSetDocValues.getValueCount());
  for (int times = 0; times < 3; times++) {
    assertTrue(sortedSetDocValues.advanceExact(0));
    assertEquals(0L, sortedSetDocValues.nextOrd());
    assertEquals(1L, sortedSetDocValues.nextOrd());
    assertEquals(2L, sortedSetDocValues.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
  }

  SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
  for (int times = 0; times < 3; times++) {
    assertTrue(sortedNumericDocValues.advanceExact(0));
    assertEquals(5, sortedNumericDocValues.docValueCount());
    assertEquals(30L, sortedNumericDocValues.nextValue());
    assertEquals(31L, sortedNumericDocValues.nextValue());
    assertEquals(32L, sortedNumericDocValues.nextValue());
    assertEquals(32L, sortedNumericDocValues.nextValue());
    assertEquals(33L, sortedNumericDocValues.nextValue());
  }

}
 

/** Counts directly from SortedNumericDocValues. */
private void countMultiValued(String field, List<MatchingDocs> matchingDocs) throws IOException {

  for (MatchingDocs hits : matchingDocs) {
    SortedNumericDocValues values = hits.context.reader().getSortedNumericDocValues(field);
    if (values == null) {
      // this field has no doc values for this segment
      continue;
    }

    NumericDocValues singleValues = DocValues.unwrapSingleton(values);

    if (singleValues != null) {
      countOneSegment(singleValues, hits);
    } else {

      DocIdSetIterator it = ConjunctionDISI.intersectIterators(
                               Arrays.asList(hits.bits.iterator(), values));
    
      for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
        int limit = values.docValueCount();
        totCount += limit;
        for (int i = 0; i < limit; i++) {
          increment(values.nextValue());
        }
      }
    }
  }
}
 

/** Wraps the provided {@link SortedNumericDocValues} in order to only select
 *  one value per parent among its {@code children} using the configured
 *  {@code selection} type. */
public static NumericDocValues wrap(SortedNumericDocValues sortedNumerics, Type selection, BitSet parents, DocIdSetIterator children) {
  NumericDocValues values;
  switch (selection) {
    case MIN:
      values = SortedNumericSelector.wrap(sortedNumerics, SortedNumericSelector.Type.MIN, SortField.Type.LONG);
      break;
    case MAX:
      values = SortedNumericSelector.wrap(sortedNumerics, SortedNumericSelector.Type.MAX, SortField.Type.LONG);
      break;
    default:
      throw new AssertionError();
  }
  return wrap(values, selection, parents, children);
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  Weight fallbackWeight = fallbackQuery.createWeight(searcher, scoreMode, boost);

  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      SortedNumericDocValues sortedNumericValues = DocValues.getSortedNumeric(context.reader(), field);
      NumericDocValues numericValues = DocValues.unwrapSingleton(sortedNumericValues);

      if (numericValues != null) {
        Sort indexSort = context.reader().getMetaData().getSort();
        if (indexSort != null
            && indexSort.getSort().length > 0
            && indexSort.getSort()[0].getField().equals(field)) {

          SortField sortField = indexSort.getSort()[0];
          DocIdSetIterator disi = getDocIdSetIterator(sortField, context, numericValues);
          return new ConstantScoreScorer(this, score(), scoreMode, disi);
        }
      }
      return fallbackWeight.scorer(context);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      // Both queries should always return the same values, so we can just check
      // if the fallback query is cacheable.
      return fallbackWeight.isCacheable(ctx);
    }
  };
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
      return new ConstantScoreScorer(this, score(), scoreMode, new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          int count = values.docValueCount();
          for(int i=0;i<count;i++) {
            if (numbers.contains(values.nextValue())) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 5; // lookup in the set
        }
      });
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return true;
    }

  };
}
 

@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
  if (merging) {
    AssertingCodec.assertThread("DocValuesProducer", creationThread);
  }
  assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;
  SortedNumericDocValues values = in.getSortedNumeric(field);
  assert values != null;
  return new AssertingLeafReader.AssertingSortedNumericDocValues(values, maxDoc);
}
 

private void writeValuesSingleBlock(SortedNumericDocValues values, long numValues, int numBitsPerValue,
    long min, long gcd, Map<Long, Integer> encode) throws IOException {
  DirectWriter writer = DirectWriter.getInstance(data, numValues, numBitsPerValue);
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      long v = values.nextValue();
      if (encode == null) {
        writer.add((v - min) / gcd);
      } else {
        writer.add(encode.get(v));
      }
    }
  }
  writer.finish();
}
 

private long writeValuesMultipleBlocks(SortedNumericDocValues values, long gcd) throws IOException {
  long[] offsets = new long[ArrayUtil.oversize(1, Long.BYTES)];
  int offsetsIndex = 0;
  final long[] buffer = new long[NUMERIC_BLOCK_SIZE];
  final ByteBuffersDataOutput encodeBuffer = ByteBuffersDataOutput.newResettableInstance();
  int upTo = 0;
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      buffer[upTo++] = values.nextValue();
      if (upTo == NUMERIC_BLOCK_SIZE) {
        offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
        offsets[offsetsIndex++] = data.getFilePointer();
        writeBlock(buffer, NUMERIC_BLOCK_SIZE, gcd, encodeBuffer);
        upTo = 0;
      }
    }
  }
  if (upTo > 0) {
    offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
    offsets[offsetsIndex++] = data.getFilePointer();
    writeBlock(buffer, upTo, gcd, encodeBuffer);
  }

  // All blocks has been written. Flush the offset jump-table
  final long offsetsOrigo = data.getFilePointer();
  for (int i = 0 ; i < offsetsIndex ; i++) {
    data.writeLong(offsets[i]);
  }
  data.writeLong(offsetsOrigo);
  return offsetsOrigo;
}
 

private void accumIntervalWithMultipleValues(SortedNumericDocValues longs) throws IOException {
  // longs should be already positioned to the correct doc
  assert longs.docID() != -1;
  final int docValueCount = longs.docValueCount();
  assert docValueCount > 0: "Should have at least one value for this document";
  int currentInterval = 0;
  for (int i = 0; i < docValueCount; i++) {
    boolean evaluateNextInterval = true;
    long value = longs.nextValue();
    while (evaluateNextInterval && currentInterval < intervals.length) {
      IntervalCompareResult result = intervals[currentInterval].includes(value);
      switch (result) {
        case INCLUDED:
          /*
           * Increment the current interval and move to the next one using
           * the same value
           */
          intervals[currentInterval].incCount();
          currentInterval++;
          break;
        case LOWER_THAN_START:
          /*
           * None of the next intervals will match this value (all of them have 
           * higher start value). Move to the next value for this document. 
           */
          evaluateNextInterval = false;
          break;
        case GREATER_THAN_END:
          /*
           * Next interval may match this value
           */
          currentInterval++;
          break;
      }
      //Maybe return if currentInterval == intervals.length?
    }
   }
}
 

/** 
 * Wraps a multi-valued SortedNumericDocValues as a single-valued view, using the specified selector 
 * and numericType.
 */
public static NumericDocValues wrap(SortedNumericDocValues sortedNumeric, Type selector, SortField.Type numericType) {
  if (numericType != SortField.Type.INT &&
      numericType != SortField.Type.LONG && 
      numericType != SortField.Type.FLOAT &&
      numericType != SortField.Type.DOUBLE) {
    throw new IllegalArgumentException("numericType must be a numeric type");
  }
  final NumericDocValues view;
  NumericDocValues singleton = DocValues.unwrapSingleton(sortedNumeric);
  if (singleton != null) {
    // it's actually single-valued in practice, but indexed as multi-valued,
    // so just sort on the underlying single-valued dv directly.
    // regardless of selector type, this optimization is safe!
    view = singleton;
  } else { 
    switch(selector) {
      case MIN: 
        view = new MinValue(sortedNumeric);
        break;
      case MAX:
        view = new MaxValue(sortedNumeric);
        break;
      default: 
        throw new AssertionError();
    }
  }
  // undo the numericutils sortability
  switch(numericType) {
    case FLOAT:
      return new FilterNumericDocValues(view) {
        @Override
        public long longValue() throws IOException {
          return NumericUtils.sortableFloatBits((int) in.longValue());
        }
      };
    case DOUBLE:
      return new FilterNumericDocValues(view) {
        @Override
        public long longValue() throws IOException {
          return NumericUtils.sortableDoubleBits(in.longValue());
        }
      };
    default:
      return view;
  }
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {

  return new ConstantScoreWeight(this, boost) {

    final Component2D tree = LatLonGeometry.create(polygons);
    final GeoEncodingUtils.PolygonPredicate polygonPredicate = GeoEncodingUtils.createComponentPredicate(tree);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            final int lon = (int) (value & 0xFFFFFFFF);
            if (polygonPredicate.test(lat, lon)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 1000f; // TODO: what should it be?
        }
      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 

MaxValue(SortedNumericDocValues in) {
  this.in = in;
}
 

MinValue(SortedNumericDocValues in) {
  this.in = in;
}
 

@Override
public SortedNumericDocValues longValues(LeafReaderContext ctx) {
    throw new UnsupportedOperationException();
}
 

protected SortedNumericDocValues getValues(ValuesSource.Numeric valuesSource, LeafReaderContext ctx) throws IOException {
    return valuesSource.longValues(ctx);
}
 

public SortedNumericDocValuesSub(MergeState.DocMap docMap, SortedNumericDocValues values) {
  super(docMap);
  this.values = values;
  assert values.docID() == -1;
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    private final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            final int lon = (int) (value & 0xFFFFFFFF);
            if (distancePredicate.test(lat, lon)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 100f; // TODO: what should it be?
        }

      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 

public Long(SortedNumericDocValues values) {
    this.values = values;
}