org.apache.lucene.document.DoubleDocValuesField Java Examples

private void addToDoc(IndexKey key, Double value){
  Preconditions.checkArgument(key.getValueType() == Double.class);
  if(value == null){
    return;
  }

  checkIfMultiValueField(key);

  final String indexFieldName = key.getIndexFieldName();
  doc.add(new DoublePoint(indexFieldName, value));
  if (key.isStored()) {
    doc.add(new StoredField(indexFieldName, value));
  }
  if (key.isSorted()) {
    Preconditions.checkArgument(key.getSortedValueType() == SearchFieldSorting.FieldType.DOUBLE);
    doc.add(new DoubleDocValuesField(indexFieldName, value));
  }
}
 

@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(500);
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  final int numDocs;
  if (TEST_NIGHTLY) {
    numDocs = TestUtil.nextInt(random(), 2049, 4000);
  } else {
    numDocs = atLeast(546);
  }
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    if (i == 545)
      document.add(new DoubleDocValuesField("onefield", LEAST_DOUBLE_VALUE));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 

@Override
public Iterable<IndexableField> convert(LuceneContext context, String path, Field field, LuceneSort annotation, Type type, Object data) {
    Collection<IndexableField> indexables = new LinkedList<>();
    Class<?> clazz = TypeUtility.getRawType(type, null);
    clazz = ClassUtility.primitiveToWrapper(clazz);
    if (Byte.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (byte) data));
        return indexables;
    }
    if (Short.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (short) data));
        return indexables;
    }
    if (Integer.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (int) data));
        return indexables;
    }
    if (Long.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (long) data));
        return indexables;
    }
    if (Float.class.isAssignableFrom(clazz)) {
        indexables.add(new FloatDocValuesField(path, (float) data));
        return indexables;
    }
    if (Double.class.isAssignableFrom(clazz)) {
        indexables.add(new DoubleDocValuesField(path, (double) data));
        return indexables;
    }
    throw new StorageException();
}
 

public void testBasicDouble() throws Exception {
  Directory d = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), d);
  Document doc = new Document();
  DoubleDocValuesField field = new DoubleDocValuesField("field", 0.0);
  doc.add(field);
  for(long l=0;l<100;l++) {
    field.setDoubleValue(l);
    w.addDocument(doc);
  }

  IndexReader r = w.getReader();

  FacetsCollector fc = new FacetsCollector();

  IndexSearcher s = newSearcher(r);
  s.search(new MatchAllDocsQuery(), fc);
  Facets facets = new DoubleRangeFacetCounts("field", fc,
      new DoubleRange("less than 10", 0.0, true, 10.0, false),
      new DoubleRange("less than or equal to 10", 0.0, true, 10.0, true),
      new DoubleRange("over 90", 90.0, false, 100.0, false),
      new DoubleRange("90 or above", 90.0, true, 100.0, false),
      new DoubleRange("over 1000", 1000.0, false, Double.POSITIVE_INFINITY, false));
                                       
  assertEquals("dim=field path=[] value=21 childCount=5\n  less than 10 (10)\n  less than or equal to 10 (11)\n  over 90 (9)\n  90 or above (10)\n  over 1000 (0)\n",
               facets.getTopChildren(10, "field").toString());
  w.close();
  IOUtils.close(r, d);
}
 

private void addLinkFields(final Random random, Document document, final String fieldName, String linkValue,
    boolean multipleValuesPerDocument, boolean globalOrdinalJoin) {
  document.add(newTextField(random, fieldName, linkValue, Field.Store.NO));

  final int linkInt = Integer.parseUnsignedInt(linkValue,16);
  document.add(new IntPoint(fieldName + "INT", linkInt));
  document.add(new FloatPoint(fieldName + "FLOAT", linkInt));

  final long linkLong = linkInt<<32 | linkInt;
  document.add(new LongPoint(fieldName + "LONG", linkLong));
  document.add(new DoublePoint(fieldName + "DOUBLE", linkLong));

  if (multipleValuesPerDocument) {
    document.add(new SortedSetDocValuesField(fieldName, new BytesRef(linkValue)));
    document.add(new SortedNumericDocValuesField(fieldName+ "INT", linkInt));
    document.add(new SortedNumericDocValuesField(fieldName+ "FLOAT", Float.floatToRawIntBits(linkInt)));
    document.add(new SortedNumericDocValuesField(fieldName+ "LONG", linkLong));
    document.add(new SortedNumericDocValuesField(fieldName+ "DOUBLE", Double.doubleToRawLongBits(linkLong)));
  } else {
    document.add(new SortedDocValuesField(fieldName, new BytesRef(linkValue)));
    document.add(new NumericDocValuesField(fieldName+ "INT", linkInt));
    document.add(new FloatDocValuesField(fieldName+ "FLOAT", linkInt));
    document.add(new NumericDocValuesField(fieldName+ "LONG", linkLong));
    document.add(new DoubleDocValuesField(fieldName+ "DOUBLE", linkLong));
  }
  if (globalOrdinalJoin) {
    document.add(new SortedDocValuesField("join_field", new BytesRef(linkValue)));
  }
}
 

private void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new StandardAnalyzer());

  for (int i = 0; i < 10; i++) {
    Document doc1 = new Document();
    doc1.add(newTextField("f1", "Apple Pie", Field.Store.YES));
    doc1.add(new SortedDocValuesField("f2", new BytesRef("a" + (i * 10 + 1))));
    doc1.add(new SortedSetDocValuesField("f3", new BytesRef("a" + (i * 10 + 1))));
    doc1.add(new NumericDocValuesField("f4", i * 10 + 1L));
    doc1.add(new FloatDocValuesField("f5", i * 10 + 1.0f));
    doc1.add(new DoubleDocValuesField("f6", i * 10 + 1.0));
    doc1.add(new SortedNumericDocValuesField("f7", i * 10 + 1L));
    doc1.add(new IntPoint("f8", i * 10 + 1));
    doc1.add(new LongPoint("f9", i * 10 + 1L));
    doc1.add(new FloatPoint("f10", i * 10 + 1.0f));
    doc1.add(new DoublePoint("f11", i * 10 + 1.0));
    writer.addDocument(doc1);

    Document doc2 = new Document();
    doc2.add(newTextField("f1", "Brownie", Field.Store.YES));
    doc2.add(new SortedDocValuesField("f2", new BytesRef("b" + (i * 10 + 2))));
    doc2.add(new SortedSetDocValuesField("f3", new BytesRef("b" + (i * 10 + 2))));
    doc2.add(new NumericDocValuesField("f4", i * 10 + 2L));
    doc2.add(new FloatDocValuesField("f5", i * 10 + 2.0f));
    doc2.add(new DoubleDocValuesField("f6", i * 10 + 2.0));
    doc2.add(new SortedNumericDocValuesField("f7", i * 10 + 2L));
    doc2.add(new IntPoint("f8", i * 10 + 2));
    doc2.add(new LongPoint("f9", i * 10 + 2L));
    doc2.add(new FloatPoint("f10", i * 10 + 2.0f));
    doc2.add(new DoublePoint("f11", i * 10 + 2.0));
    writer.addDocument(doc2);

    Document doc3 = new Document();
    doc3.add(newTextField("f1", "Chocolate Pie", Field.Store.YES));
    doc3.add(new SortedDocValuesField("f2", new BytesRef("c" + (i * 10 + 3))));
    doc3.add(new SortedSetDocValuesField("f3", new BytesRef("c" + (i * 10 + 3))));
    doc3.add(new NumericDocValuesField("f4", i * 10 + 3L));
    doc3.add(new FloatDocValuesField("f5", i * 10 + 3.0f));
    doc3.add(new DoubleDocValuesField("f6", i * 10 + 3.0));
    doc3.add(new SortedNumericDocValuesField("f7", i * 10 + 3L));
    doc3.add(new IntPoint("f8", i * 10 + 3));
    doc3.add(new LongPoint("f9", i * 10 + 3L));
    doc3.add(new FloatPoint("f10", i * 10 + 3.0f));
    doc3.add(new DoublePoint("f11", i * 10 + 3.0));
    writer.addDocument(doc3);

    Document doc4 = new Document();
    doc4.add(newTextField("f1", "Doughnut", Field.Store.YES));
    doc4.add(new SortedDocValuesField("f2", new BytesRef("d" + (i * 10 + 4))));
    doc4.add(new SortedSetDocValuesField("f3", new BytesRef("d" + (i * 10 + 4))));
    doc4.add(new NumericDocValuesField("f4", i * 10 + 4L));
    doc4.add(new FloatDocValuesField("f5", i * 10 + 4.0f));
    doc4.add(new DoubleDocValuesField("f6", i * 10 + 4.0));
    doc4.add(new SortedNumericDocValuesField("f7", i * 10 + 4L));
    doc4.add(new IntPoint("f8", i * 10 + 4));
    doc4.add(new LongPoint("f9", i * 10 + 4L));
    doc4.add(new FloatPoint("f10", i * 10 + 4.0f));
    doc4.add(new DoublePoint("f11", i * 10 + 4.0));
    writer.addDocument(doc4);

    Document doc5 = new Document();
    doc5.add(newTextField("f1", "Eclair", Field.Store.YES));
    doc5.add(new SortedDocValuesField("f2", new BytesRef("e" + (i * 10 + 5))));
    doc5.add(new SortedSetDocValuesField("f3", new BytesRef("e" + (i * 10 + 5))));
    doc5.add(new NumericDocValuesField("f4", i * 10 + 5L));
    doc5.add(new FloatDocValuesField("f5", i * 10 + 5.0f));
    doc5.add(new DoubleDocValuesField("f6", i * 10 + 5.0));
    doc5.add(new SortedNumericDocValuesField("f7", i * 10 + 5L));
    doc5.add(new IntPoint("f8", i * 10 + 5));
    doc5.add(new LongPoint("f9", i * 10 + 5L));
    doc5.add(new FloatPoint("f10", i * 10 + 5.0f));
    doc5.add(new DoublePoint("f11", i * 10 + 5.0));
    writer.addDocument(doc5);
  }
  writer.commit();
  writer.close();
  dir.close();
}
 

public void testDocsWithDoubleValues() throws IOException {
  try (Directory dir = newDirectory();
      IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig())) {
    String field = "numeric";
    int numDocs = TestUtil.nextInt(random(), 1, 100);
    double[] docValues = new double[numDocs];
    double nextVal = 1.0;
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      if (random().nextBoolean()) { // not all documents have a value
        doc.add(new DoubleDocValuesField(field, nextVal));
        doc.add(new StringField("id", "doc" + i, Store.NO));
        docValues[i] = nextVal;
        ++nextVal;
      }
      indexWriter.addDocument(doc);
    }

    // 20% of cases delete some docs
    if (random().nextDouble() < 0.2) {
      for (int i = 0; i < numDocs; i++) {
        if (random().nextBoolean()) {
          indexWriter.deleteDocuments(new Term("id", "doc" + i));
          docValues[i] = 0;
        }
      }
    }

    try (DirectoryReader reader = DirectoryReader.open(indexWriter)) {
      IndexSearcher searcher = new IndexSearcher(reader);
      DoubleDocValuesStats stats = new DoubleDocValuesStats(field);
      searcher.search(new MatchAllDocsQuery(), new DocValuesStatsCollector(stats));

      int expCount = (int) Arrays.stream(docValues).filter(v -> v > 0).count();
      assertEquals(expCount, stats.count());
      int numDocsWithoutField = (int) getZeroValues(docValues).count();
      assertEquals(computeExpMissing(numDocsWithoutField, numDocs, reader), stats.missing());
      if (stats.count() > 0) {
        DoubleSummaryStatistics sumStats = getPositiveValues(docValues).summaryStatistics();
        assertEquals(sumStats.getMax(), stats.max().doubleValue(), 0.00001);
        assertEquals(sumStats.getMin(), stats.min().doubleValue(), 0.00001);
        assertEquals(sumStats.getAverage(), stats.mean(), 0.00001);
        assertEquals(sumStats.getSum(), stats.sum(), 0.00001);
        double variance = computeVariance(docValues, stats.mean, stats.count());
        assertEquals(variance, stats.variance(), 0.00001);
        assertEquals(Math.sqrt(variance), stats.stdev(), 0.00001);
      }
    }
  }
}
 

private void addDoc(IndexWriter writer, int id) throws IOException
{
  Document doc = new Document();
  doc.add(new TextField("content", "aaa", Field.Store.NO));
  doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));
  FieldType customType2 = new FieldType(TextField.TYPE_STORED);
  customType2.setStoreTermVectors(true);
  customType2.setStoreTermVectorPositions(true);
  customType2.setStoreTermVectorOffsets(true);
  doc.add(new Field("autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
  doc.add(new Field("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
  doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2));
  doc.add(new Field("fie\u2C77ld", "field with non-ascii name", customType2));

  // add docvalues fields
  doc.add(new NumericDocValuesField("dvByte", (byte) id));
  byte bytes[] = new byte[] {
    (byte)(id >>> 24), (byte)(id >>> 16),(byte)(id >>> 8),(byte)id
  };
  BytesRef ref = new BytesRef(bytes);
  doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));
  doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));
  doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));
  doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));
  doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));
  doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));
  doc.add(new DoubleDocValuesField("dvDouble", (double)id));
  doc.add(new FloatDocValuesField("dvFloat", (float)id));
  doc.add(new NumericDocValuesField("dvInt", id));
  doc.add(new NumericDocValuesField("dvLong", id));
  doc.add(new NumericDocValuesField("dvPacked", id));
  doc.add(new NumericDocValuesField("dvShort", (short)id));
  doc.add(new SortedSetDocValuesField("dvSortedSet", ref));
  doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id));

  doc.add(new IntPoint("intPoint1d", id));
  doc.add(new IntPoint("intPoint2d", id, 2*id));
  doc.add(new FloatPoint("floatPoint1d", (float) id));
  doc.add(new FloatPoint("floatPoint2d", (float) id, (float) 2*id));
  doc.add(new LongPoint("longPoint1d", id));
  doc.add(new LongPoint("longPoint2d", id, 2*id));
  doc.add(new DoublePoint("doublePoint1d", (double) id));
  doc.add(new DoublePoint("doublePoint2d", (double) id, (double) 2*id));
  doc.add(new BinaryPoint("binaryPoint1d", bytes));
  doc.add(new BinaryPoint("binaryPoint2d", bytes, bytes));
  
  // a field with both offsets and term vectors for a cross-check
  FieldType customType3 = new FieldType(TextField.TYPE_STORED);
  customType3.setStoreTermVectors(true);
  customType3.setStoreTermVectorPositions(true);
  customType3.setStoreTermVectorOffsets(true);
  customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3));
  // a field that omits only positions
  FieldType customType4 = new FieldType(TextField.TYPE_STORED);
  customType4.setStoreTermVectors(true);
  customType4.setStoreTermVectorPositions(false);
  customType4.setStoreTermVectorOffsets(true);
  customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));
  // TODO: 
  //   index different norms types via similarity (we use a random one currently?!)
  //   remove any analyzer randomness, explicitly add payloads for certain fields.
  writer.addDocument(doc);
}