org.apache.lucene.document.FloatDocValuesField Java Examples

@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(500);
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  final int numDocs;
  if (TEST_NIGHTLY) {
    numDocs = TestUtil.nextInt(random(), 2049, 4000);
  } else {
    numDocs = atLeast(546);
  }
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    if (i == 545)
      document.add(new DoubleDocValuesField("onefield", LEAST_DOUBLE_VALUE));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 

@Override
public Iterable<IndexableField> convert(LuceneContext context, String path, Field field, LuceneSort annotation, Type type, Object data) {
    Collection<IndexableField> indexables = new LinkedList<>();
    Class<?> clazz = TypeUtility.getRawType(type, null);
    clazz = ClassUtility.primitiveToWrapper(clazz);
    if (Byte.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (byte) data));
        return indexables;
    }
    if (Short.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (short) data));
        return indexables;
    }
    if (Integer.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (int) data));
        return indexables;
    }
    if (Long.class.isAssignableFrom(clazz)) {
        indexables.add(new NumericDocValuesField(path, (long) data));
        return indexables;
    }
    if (Float.class.isAssignableFrom(clazz)) {
        indexables.add(new FloatDocValuesField(path, (float) data));
        return indexables;
    }
    if (Double.class.isAssignableFrom(clazz)) {
        indexables.add(new DoubleDocValuesField(path, (double) data));
        return indexables;
    }
    throw new StorageException();
}
 

private void addLinkFields(final Random random, Document document, final String fieldName, String linkValue,
    boolean multipleValuesPerDocument, boolean globalOrdinalJoin) {
  document.add(newTextField(random, fieldName, linkValue, Field.Store.NO));

  final int linkInt = Integer.parseUnsignedInt(linkValue,16);
  document.add(new IntPoint(fieldName + "INT", linkInt));
  document.add(new FloatPoint(fieldName + "FLOAT", linkInt));

  final long linkLong = linkInt<<32 | linkInt;
  document.add(new LongPoint(fieldName + "LONG", linkLong));
  document.add(new DoublePoint(fieldName + "DOUBLE", linkLong));

  if (multipleValuesPerDocument) {
    document.add(new SortedSetDocValuesField(fieldName, new BytesRef(linkValue)));
    document.add(new SortedNumericDocValuesField(fieldName+ "INT", linkInt));
    document.add(new SortedNumericDocValuesField(fieldName+ "FLOAT", Float.floatToRawIntBits(linkInt)));
    document.add(new SortedNumericDocValuesField(fieldName+ "LONG", linkLong));
    document.add(new SortedNumericDocValuesField(fieldName+ "DOUBLE", Double.doubleToRawLongBits(linkLong)));
  } else {
    document.add(new SortedDocValuesField(fieldName, new BytesRef(linkValue)));
    document.add(new NumericDocValuesField(fieldName+ "INT", linkInt));
    document.add(new FloatDocValuesField(fieldName+ "FLOAT", linkInt));
    document.add(new NumericDocValuesField(fieldName+ "LONG", linkLong));
    document.add(new DoubleDocValuesField(fieldName+ "DOUBLE", linkLong));
  }
  if (globalOrdinalJoin) {
    document.add(new SortedDocValuesField("join_field", new BytesRef(linkValue)));
  }
}
 

public void testFloatSortOptimization() throws IOException {
  final Directory dir = newDirectory();
  final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());
  final int numDocs = atLeast(10000);
  for (int i = 0; i < numDocs; ++i) {
    final Document doc = new Document();
    float f = 1f * i;
    doc.add(new FloatDocValuesField("my_field", f));
    doc.add(new FloatPoint("my_field", i));
    writer.addDocument(doc);
  }
  final IndexReader reader = DirectoryReader.open(writer);
  IndexSearcher searcher = new IndexSearcher(reader);
  final SortField sortField = new SortField("my_field", SortField.Type.FLOAT);
  final Sort sort = new Sort(sortField);
  final int numHits = 3;
  final int totalHitsThreshold = 3;

  { // simple sort
    final TopFieldCollector collector = TopFieldCollector.create(sort, numHits, null, totalHitsThreshold);
    searcher.search(new MatchAllDocsQuery(), collector);
    TopDocs topDocs = collector.topDocs();
    assertEquals(topDocs.scoreDocs.length, numHits);
    for (int i = 0; i < numHits; i++) {
      FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i];
      assertEquals(1f * i, fieldDoc.fields[0]);
    }
    assertTrue(collector.isEarlyTerminated());
    assertTrue(topDocs.totalHits.value < numDocs);
  }

  writer.close();
  reader.close();
  dir.close();
}
 

public static Document buildDoc(String text, float value) throws IOException {
    String id = UUID.randomUUID().toString();
    Document d = new Document();
    d.add(newStringField("id", id, Field.Store.YES));
    d.add(newStringField("text", text, Field.Store.NO));
    d.add(new FloatDocValuesField("score", value));
    return d;
}
 

private void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new StandardAnalyzer());

  for (int i = 0; i < 10; i++) {
    Document doc1 = new Document();
    doc1.add(newTextField("f1", "Apple Pie", Field.Store.YES));
    doc1.add(new SortedDocValuesField("f2", new BytesRef("a" + (i * 10 + 1))));
    doc1.add(new SortedSetDocValuesField("f3", new BytesRef("a" + (i * 10 + 1))));
    doc1.add(new NumericDocValuesField("f4", i * 10 + 1L));
    doc1.add(new FloatDocValuesField("f5", i * 10 + 1.0f));
    doc1.add(new DoubleDocValuesField("f6", i * 10 + 1.0));
    doc1.add(new SortedNumericDocValuesField("f7", i * 10 + 1L));
    doc1.add(new IntPoint("f8", i * 10 + 1));
    doc1.add(new LongPoint("f9", i * 10 + 1L));
    doc1.add(new FloatPoint("f10", i * 10 + 1.0f));
    doc1.add(new DoublePoint("f11", i * 10 + 1.0));
    writer.addDocument(doc1);

    Document doc2 = new Document();
    doc2.add(newTextField("f1", "Brownie", Field.Store.YES));
    doc2.add(new SortedDocValuesField("f2", new BytesRef("b" + (i * 10 + 2))));
    doc2.add(new SortedSetDocValuesField("f3", new BytesRef("b" + (i * 10 + 2))));
    doc2.add(new NumericDocValuesField("f4", i * 10 + 2L));
    doc2.add(new FloatDocValuesField("f5", i * 10 + 2.0f));
    doc2.add(new DoubleDocValuesField("f6", i * 10 + 2.0));
    doc2.add(new SortedNumericDocValuesField("f7", i * 10 + 2L));
    doc2.add(new IntPoint("f8", i * 10 + 2));
    doc2.add(new LongPoint("f9", i * 10 + 2L));
    doc2.add(new FloatPoint("f10", i * 10 + 2.0f));
    doc2.add(new DoublePoint("f11", i * 10 + 2.0));
    writer.addDocument(doc2);

    Document doc3 = new Document();
    doc3.add(newTextField("f1", "Chocolate Pie", Field.Store.YES));
    doc3.add(new SortedDocValuesField("f2", new BytesRef("c" + (i * 10 + 3))));
    doc3.add(new SortedSetDocValuesField("f3", new BytesRef("c" + (i * 10 + 3))));
    doc3.add(new NumericDocValuesField("f4", i * 10 + 3L));
    doc3.add(new FloatDocValuesField("f5", i * 10 + 3.0f));
    doc3.add(new DoubleDocValuesField("f6", i * 10 + 3.0));
    doc3.add(new SortedNumericDocValuesField("f7", i * 10 + 3L));
    doc3.add(new IntPoint("f8", i * 10 + 3));
    doc3.add(new LongPoint("f9", i * 10 + 3L));
    doc3.add(new FloatPoint("f10", i * 10 + 3.0f));
    doc3.add(new DoublePoint("f11", i * 10 + 3.0));
    writer.addDocument(doc3);

    Document doc4 = new Document();
    doc4.add(newTextField("f1", "Doughnut", Field.Store.YES));
    doc4.add(new SortedDocValuesField("f2", new BytesRef("d" + (i * 10 + 4))));
    doc4.add(new SortedSetDocValuesField("f3", new BytesRef("d" + (i * 10 + 4))));
    doc4.add(new NumericDocValuesField("f4", i * 10 + 4L));
    doc4.add(new FloatDocValuesField("f5", i * 10 + 4.0f));
    doc4.add(new DoubleDocValuesField("f6", i * 10 + 4.0));
    doc4.add(new SortedNumericDocValuesField("f7", i * 10 + 4L));
    doc4.add(new IntPoint("f8", i * 10 + 4));
    doc4.add(new LongPoint("f9", i * 10 + 4L));
    doc4.add(new FloatPoint("f10", i * 10 + 4.0f));
    doc4.add(new DoublePoint("f11", i * 10 + 4.0));
    writer.addDocument(doc4);

    Document doc5 = new Document();
    doc5.add(newTextField("f1", "Eclair", Field.Store.YES));
    doc5.add(new SortedDocValuesField("f2", new BytesRef("e" + (i * 10 + 5))));
    doc5.add(new SortedSetDocValuesField("f3", new BytesRef("e" + (i * 10 + 5))));
    doc5.add(new NumericDocValuesField("f4", i * 10 + 5L));
    doc5.add(new FloatDocValuesField("f5", i * 10 + 5.0f));
    doc5.add(new DoubleDocValuesField("f6", i * 10 + 5.0));
    doc5.add(new SortedNumericDocValuesField("f7", i * 10 + 5L));
    doc5.add(new IntPoint("f8", i * 10 + 5));
    doc5.add(new LongPoint("f9", i * 10 + 5L));
    doc5.add(new FloatPoint("f10", i * 10 + 5.0f));
    doc5.add(new DoublePoint("f11", i * 10 + 5.0));
    writer.addDocument(doc5);
  }
  writer.commit();
  writer.close();
  dir.close();
}
 

@Override
public void setUp() throws Exception {
  super.setUp();

  // populate an index with documents - artist, song and weeksAtNumberOne
  dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();

  Field yearField = newTextField("year", "", Field.Store.NO);
  SortedDocValuesField artistField = new SortedDocValuesField("artist",
      new BytesRef(""));
  Field weeksAtNumberOneField = new FloatDocValuesField("weeksAtNumberOne",
      0.0F);
  Field weeksStoredField = new StoredField("weeks", 0.0F);
  Field idField = newStringField("id", "", Field.Store.YES);
  Field songField = newTextField("song", "", Field.Store.NO);
  Field storedArtistField = newTextField("artistName", "", Field.Store.NO);

  doc.add(idField);
  doc.add(weeksAtNumberOneField);
  doc.add(storedArtistField);
  doc.add(songField);
  doc.add(weeksStoredField);
  doc.add(yearField);
  doc.add(artistField);

  parsedRecords.clear();
  for (int i = 0; i < hitsOfThe60s.length; i++) {
    String cols[] = hitsOfThe60s[i].split("\t");
    Record record = new Record(String.valueOf(i), cols[0], cols[1], cols[2],
        Float.parseFloat(cols[3]));
    parsedRecords.put(record.id, record);
    idField.setStringValue(record.id);
    yearField.setStringValue(record.year);
    storedArtistField.setStringValue(record.artist);
    artistField.setBytesValue(new BytesRef(record.artist));
    songField.setStringValue(record.song);
    weeksStoredField.setFloatValue(record.weeks);
    weeksAtNumberOneField.setFloatValue(record.weeks);
    writer.addDocument(doc);
    if (i % 10 == 0) {
      // Causes the creation of multiple segments for our test
      writer.commit();
    }
  }
  reader = writer.getReader();
  writer.close();
  searcher = newSearcher(reader);
  artistDocValues = MultiDocValues.getSortedValues(reader, "artist");
}
 

private void addDoc(IndexWriter writer, int id) throws IOException
{
  Document doc = new Document();
  doc.add(new TextField("content", "aaa", Field.Store.NO));
  doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));
  FieldType customType2 = new FieldType(TextField.TYPE_STORED);
  customType2.setStoreTermVectors(true);
  customType2.setStoreTermVectorPositions(true);
  customType2.setStoreTermVectorOffsets(true);
  doc.add(new Field("autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
  doc.add(new Field("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
  doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2));
  doc.add(new Field("fie\u2C77ld", "field with non-ascii name", customType2));

  // add docvalues fields
  doc.add(new NumericDocValuesField("dvByte", (byte) id));
  byte bytes[] = new byte[] {
    (byte)(id >>> 24), (byte)(id >>> 16),(byte)(id >>> 8),(byte)id
  };
  BytesRef ref = new BytesRef(bytes);
  doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));
  doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));
  doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));
  doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));
  doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));
  doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));
  doc.add(new DoubleDocValuesField("dvDouble", (double)id));
  doc.add(new FloatDocValuesField("dvFloat", (float)id));
  doc.add(new NumericDocValuesField("dvInt", id));
  doc.add(new NumericDocValuesField("dvLong", id));
  doc.add(new NumericDocValuesField("dvPacked", id));
  doc.add(new NumericDocValuesField("dvShort", (short)id));
  doc.add(new SortedSetDocValuesField("dvSortedSet", ref));
  doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id));

  doc.add(new IntPoint("intPoint1d", id));
  doc.add(new IntPoint("intPoint2d", id, 2*id));
  doc.add(new FloatPoint("floatPoint1d", (float) id));
  doc.add(new FloatPoint("floatPoint2d", (float) id, (float) 2*id));
  doc.add(new LongPoint("longPoint1d", id));
  doc.add(new LongPoint("longPoint2d", id, 2*id));
  doc.add(new DoublePoint("doublePoint1d", (double) id));
  doc.add(new DoublePoint("doublePoint2d", (double) id, (double) 2*id));
  doc.add(new BinaryPoint("binaryPoint1d", bytes));
  doc.add(new BinaryPoint("binaryPoint2d", bytes, bytes));
  
  // a field with both offsets and term vectors for a cross-check
  FieldType customType3 = new FieldType(TextField.TYPE_STORED);
  customType3.setStoreTermVectors(true);
  customType3.setStoreTermVectorPositions(true);
  customType3.setStoreTermVectorOffsets(true);
  customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3));
  // a field that omits only positions
  FieldType customType4 = new FieldType(TextField.TYPE_STORED);
  customType4.setStoreTermVectors(true);
  customType4.setStoreTermVectorPositions(false);
  customType4.setStoreTermVectorOffsets(true);
  customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));
  // TODO: 
  //   index different norms types via similarity (we use a random one currently?!)
  //   remove any analyzer randomness, explicitly add payloads for certain fields.
  writer.addDocument(doc);
}