org.apache.lucene.util.NumericUtils Java Examples

private BytesRef valueForSearch(Object value) {
    if (value == null) return null;
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(parseValue(value), 0, bytesRef); // 0 because of exact match
    return bytesRef.get();
}
 

@Override
public BytesRef indexedValueForSearch(Object value) {
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(parseValue(value), 0, bytesRef); // 0 because of exact match
    return bytesRef.get();
}
 

private FieldComparator<?> getFloatComparator(int numHits) {
  return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableFloatBits((int) super.longValue());
        }
      };
    }
  };
}
 

@Override
public SortField readSortField(DataInput in) throws IOException {
  SortedNumericSortField sf = new SortedNumericSortField(in.readString(), readType(in), in.readInt() == 1, readSelectorType(in));
  if (in.readInt() == 1) {
    switch (sf.type) {
      case INT:
        sf.setMissingValue(in.readInt());
        break;
      case LONG:
        sf.setMissingValue(in.readLong());
        break;
      case FLOAT:
        sf.setMissingValue(NumericUtils.sortableIntToFloat(in.readInt()));
        break;
      case DOUBLE:
        sf.setMissingValue(NumericUtils.sortableLongToDouble(in.readLong()));
        break;
      default:
        throw new AssertionError();
    }
  }
  return sf;
}
 

@Override
public ComparableProvider[] getComparableProviders(List<? extends LeafReader> readers) throws IOException {
  ComparableProvider[] providers = new ComparableProvider[readers.size()];
  final float missingValue;
  if (this.missingValue != null) {
    missingValue = this.missingValue;
  } else {
    missingValue = 0.0f;
  }

  for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
    final NumericDocValues values = valuesProvider.get(readers.get(readerIndex));

    providers[readerIndex] = docID -> {
      float value = missingValue;
      if (values.advanceExact(docID)) {
        value = Float.intBitsToFloat((int) values.longValue());
      }
      return NumericUtils.floatToSortableInt(value);
    };
  }
  return providers;
}
 

public void testFloatFieldMinMax() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(100);
  float minValue = Float.POSITIVE_INFINITY;
  float maxValue = Float.NEGATIVE_INFINITY;
  for(int i=0;i<numDocs;i++ ){
    Document doc = new Document();
    float num = random().nextFloat();
    minValue = Math.min(num, minValue);
    maxValue = Math.max(num, maxValue);
    doc.add(new LegacyFloatField("field", num, Field.Store.NO));
    w.addDocument(doc);
  }
  
  IndexReader r = w.getReader();
  Terms terms = MultiTerms.getTerms(r, "field");
  assertEquals(minValue, NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMinInt(terms)), 0.0f);
  assertEquals(maxValue, NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMaxInt(terms)), 0.0f);

  r.close();
  w.close();
  dir.close();
}
 

protected Query getRangeQueryForMultiValuedFloatDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) {
  float minVal,maxVal;
  if (min == null) {
    minVal = Float.NEGATIVE_INFINITY;
  } else {
    minVal = parseFloatFromUser(sf.getName(), min);
    if (!minInclusive) {
      if (minVal == Float.POSITIVE_INFINITY) return new MatchNoDocsQuery();
      minVal = FloatPoint.nextUp(minVal);
    }
  }
  if (max == null) {
    maxVal = Float.POSITIVE_INFINITY;
  } else {
    maxVal = parseFloatFromUser(sf.getName(), max);
    if (!maxInclusive) {
      if (maxVal == Float.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
      maxVal = FloatPoint.nextDown(maxVal);
    }
  }
  Long minBits = (long)NumericUtils.floatToSortableInt(minVal);
  Long maxBits = (long)NumericUtils.floatToSortableInt(maxVal);
  return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, true, true, true);
}
 

public void testDoubles() throws Exception {
  double[] vals=new double[]{
    Double.NEGATIVE_INFINITY, -2.3E25, -1.0E15, -1.0, -1.0E-1, -1.0E-2, -0.0, 
    +0.0, 1.0E-2, 1.0E-1, 1.0, 1.0E15, 2.3E25, Double.POSITIVE_INFINITY, Double.NaN
  };
  long[] longVals=new long[vals.length];
  
  // check forward and back conversion
  for (int i=0; i<vals.length; i++) {
    longVals[i]= NumericUtils.doubleToSortableLong(vals[i]);
    assertTrue( "forward and back conversion should generate same double", Double.compare(vals[i], NumericUtils.sortableLongToDouble(longVals[i]))==0 );
  }
  
  // check sort order (prefixVals should be ascending)
  for (int i=1; i<longVals.length; i++) {
    assertTrue( "check sort order", longVals[i-1] < longVals[i] );
  }
}
 

@Override
public BytesRef indexedValueForSearch(Object value) {
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(parseLongValue(value), 0, bytesRef);  // 0 because of exact match
    return bytesRef.get();
}
 

static LongFunction<Object> bitsToValue(FieldType fieldType) {
  switch (fieldType.getNumberType()) {
    case LONG:
      return (bits)-> bits;
    case DATE:
      return (bits)-> new Date(bits);
    case INTEGER:
      return (bits)-> (int)bits;
    case FLOAT:
      return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
    case DOUBLE:
      return (bits)-> NumericUtils.sortableLongToDouble(bits);
    default:
      throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
  }
}
 

@Override
public BytesRef next() {
  BytesRefBuilder b = new BytesRefBuilder();
  NumericUtils.longToPrefixCoded((int) values.valueAt(this.count++), 0, b);
  return b.toBytesRef();
}
 

@Override
public Object toObject(SchemaField sf, BytesRef term) {
  switch (type) {
    case INTEGER:
      return LegacyNumericUtils.prefixCodedToInt(term);
    case FLOAT:
      return NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(term));
    case LONG:
      return LegacyNumericUtils.prefixCodedToLong(term);
    case DOUBLE:
      return NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(term));
    case DATE:
      return new Date(LegacyNumericUtils.prefixCodedToLong(term));
    default:
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
  }
}
 

@Override
public String indexedToReadable(String _indexedForm) {
  final BytesRef indexedForm = new BytesRef(_indexedForm);
  switch (type) {
    case INTEGER:
      return Integer.toString( LegacyNumericUtils.prefixCodedToInt(indexedForm) );
    case FLOAT:
      return Float.toString( NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(indexedForm)) );
    case LONG:
      return Long.toString( LegacyNumericUtils.prefixCodedToLong(indexedForm) );
    case DOUBLE:
      return Double.toString( NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(indexedForm)) );
    case DATE:
      return Instant.ofEpochMilli(LegacyNumericUtils.prefixCodedToLong(indexedForm)).toString();
    default:
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
  }
}
 

@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    float minValue = NumericUtils.sortableIntToFloat(NumericUtils.getMinInt(terms));
    float maxValue = NumericUtils.sortableIntToFloat(NumericUtils.getMaxInt(terms));
    return new FieldStats.Float(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 

public void testSortableFloatNaN() {
  final int plusInf = NumericUtils.floatToSortableInt(Float.POSITIVE_INFINITY);
  for (float nan : FLOAT_NANs) {
    assertTrue(Float.isNaN(nan));
    final int sortable = NumericUtils.floatToSortableInt(nan);
    assertTrue("Float not sorted correctly: " + nan + ", int repr: " 
        + sortable + ", positive inf.: " + plusInf, sortable > plusInf);
  }
}
 

/**
 * Pick the next dimension to split.
 * @param minPackedValue the min values for all dimensions
 * @param maxPackedValue the max values for all dimensions
 * @param parentSplits how many times each dim has been split on the parent levels
 * @return the dimension to split
 */
protected int split(byte[] minPackedValue, byte[] maxPackedValue, int[] parentSplits) {
  // First look at whether there is a dimension that has split less than 2x less than
  // the dim that has most splits, and return it if there is such a dimension and it
  // does not only have equals values. This helps ensure all dimensions are indexed.
  int maxNumSplits = 0;
  for (int numSplits : parentSplits) {
    maxNumSplits = Math.max(maxNumSplits, numSplits);
  }
  for (int dim = 0; dim < numIndexDims; ++dim) {
    final int offset = dim * bytesPerDim;
    if (parentSplits[dim] < maxNumSplits / 2 &&
        Arrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) != 0) {
      return dim;
    }
  }

  // Find which dim has the largest span so we can split on it:
  int splitDim = -1;
  for(int dim=0;dim<numIndexDims;dim++) {
    NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
    if (splitDim == -1 || Arrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
      System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
      splitDim = dim;
    }
  }

  //System.out.println("SPLIT: " + splitDim);
  return splitDim;
}
 

@Override
public void readableToIndexed(CharSequence val, BytesRefBuilder result) {
  final String s = val.toString();
  if (s == null)
    return;

  result.grow(Integer.BYTES);
  result.setLength(Integer.BYTES);
  final Integer intValue = enumMapping.stringValueToIntValue(s);
  NumericUtils.intToSortableBytes(intValue, result.bytes(), 0);
}
 

/** encodes a two-dimensional geopoint (lat, lon) into a byte array */
static void encode(double lat, double lon, byte[] result, int offset) {
  if (result == null) {
    result = new byte[BYTES*4];
  }
  NumericUtils.intToSortableBytes(encodeLatitude(lat), result, offset);
  NumericUtils.intToSortableBytes(encodeLongitude(lon), result, offset + BYTES);
}
 

@Override
public void apply(SumDoubleState state, int doc) throws IOException {
    if (values.advanceExact(doc) && values.docValueCount() == 1) {
        state.sum += NumericUtils.sortableLongToDouble(values.nextValue());
        state.hadValue = true;
    }
}
 

@Override
protected void insertValues(IndexWriter writer) throws Exception {
    for (float f = -0.5f; f < 10.0f; f++) {
        Document doc = new Document();
        doc.add(new SortedNumericDocValuesField(column, NumericUtils.floatToSortableInt(f)));
        writer.addDocument(doc);
    }
}
 

@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinLong(terms);
    long maxValue = NumericUtils.getMaxLong(terms);
    return new FieldStats.Date(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue, dateTimeFormatter()
    );
}
 

@Override
public BytesRef indexedValueForSearch(Object value) {
    int intValue = NumericUtils.floatToSortableInt(parseValue(value));
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.intToPrefixCoded(intValue, 0, bytesRef);   // 0 because of exact match
    return bytesRef.get();
}
 

/**
 * Return the {@link InetAddress} that compares immediately greater than
 * {@code address}.
 * @throws ArithmeticException if the provided address is the
 *              {@link #MAX_VALUE maximum ip address}
 */
public static InetAddress nextUp(InetAddress address) {
  if (address.equals(MAX_VALUE)) {
    throw new ArithmeticException("Overflow: there is no greater InetAddress than "
        + address.getHostAddress());
  }
  byte[] delta = new byte[BYTES];
  delta[BYTES-1] = 1;
  byte[] nextUpBytes = new byte[InetAddressPoint.BYTES];
  NumericUtils.add(InetAddressPoint.BYTES, 0, encode(address), delta, nextUpBytes);
  return decode(nextUpBytes);
}
 

@Override
public BytesRef indexedValueForSearch(Object value) {
    long longValue = NumericUtils.doubleToSortableLong(parseDoubleValue(value));
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(longValue, 0, bytesRef);   // 0 because of exact match
    return bytesRef.get();
}
 

@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinLong(terms);
    long maxValue = NumericUtils.getMaxLong(terms);
    return new FieldStats.Long(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 

protected int split(byte[] minPackedValue, byte[] maxPackedValue) {
  // Find which dim has the largest span so we can split on it:
  int splitDim = -1;
  for(int dim=0;dim<numIndexDims;dim++) {
    NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
    if (splitDim == -1 || Arrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
      System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
      splitDim = dim;
    }
  }

  //System.out.println("SPLIT: " + splitDim);
  return splitDim;
}
 

@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinInt(terms);
    long maxValue = NumericUtils.getMaxInt(terms);
    return new FieldStats.Long(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer);
  iwConfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);
  for (String [] doc : documents) {
    Document document = new Document();
    document.add(new StringField("id", doc[0], Field.Store.NO));
    document.add(new SortedDocValuesField("id", new BytesRef(doc[0])));
    document.add(new NumericDocValuesField("double", Double.doubleToRawLongBits(Double.parseDouble(doc[1]))));
    document.add(new NumericDocValuesField("float", Float.floatToRawIntBits(Float.parseFloat(doc[2]))));
    document.add(new NumericDocValuesField("int", Integer.parseInt(doc[3])));
    document.add(new NumericDocValuesField("long", Long.parseLong(doc[4])));
    document.add(new StringField("string", doc[5], Field.Store.NO));
    document.add(new SortedDocValuesField("string", new BytesRef(doc[5])));
    document.add(new TextField("text", doc[6], Field.Store.NO));
    document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[7]))));
    document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[8]))));
    document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[9]))));
    document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[7]))));
    document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[8]))));
    document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[9]))));
    document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[10])));
    document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[11])));
    document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[12])));
    document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[10])));
    document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[11])));
    document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[12])));
    iw.addDocument(document);
  }
  
  reader = iw.getReader();
  searcher = newSearcher(reader);
  iw.close();
}
 

@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinInt(terms);
    long maxValue = NumericUtils.getMaxInt(terms);
    return new FieldStats.Long(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 

private Number toCorrectType(long value) {
  switch (numberType) {
    case INTEGER:
    case LONG:
      return value;
    case FLOAT:
      return NumericUtils.sortableIntToFloat((int)value);
    case DOUBLE:
      return NumericUtils.sortableLongToDouble(value);
    default:
      throw new AssertionError("Unsupported number type");
  }
}