Java Code Examples for org.apache.lucene.index.NumericDocValues#longValue()

private void writeValues(NumericDocValues values, int numBytesPerValue, IndexOutput out) throws IOException, AssertionError {
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    long value = values.longValue();
    switch (numBytesPerValue) {
      case 1:
        out.writeByte((byte) value);
        break;
      case 2:
        out.writeShort((short) value);
        break;
      case 4:
        out.writeInt((int) value);
        break;
      case 8:
        out.writeLong(value);
        break;
      default:
        throw new AssertionError();
    }
  }
}
 

public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
  int val;
  SortValue sortValue = sortDoc.getSortValue(this.field);
  if (sortValue != null) {
    if (sortValue.isPresent()) {
      val = (int) sortValue.getCurrentValue();
    } else { //empty-value
      return false;
    }
  } else {
    // field is not part of 'sort' param, but part of 'fl' param
    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
    if (vals.advance(sortDoc.docId) == sortDoc.docId) {
      val = (int) vals.longValue();
    } else {
      return false;
    }
  }
  ew.put(this.field, val);
  return true;
}
 

public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
  Long val;
  SortValue sortValue = sortDoc.getSortValue(this.field);
  if (sortValue != null) {
    if (sortValue.isPresent()) {
      val = (long) sortValue.getCurrentValue();
    } else { //empty-value
      return false;
    }
  } else {
    // field is not part of 'sort' param, but part of 'fl' param
    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
    if (vals.advance(sortDoc.docId) == sortDoc.docId) {
      val = vals.longValue();
    } else {
      return false;
    }
  }
  ew.put(this.field, new Date(val));
  return true;
}
 

public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
  long val;
  SortValue sortValue = sortDoc.getSortValue(this.field);
  if (sortValue != null) {
    if (sortValue.isPresent()) {
      val = (long) sortValue.getCurrentValue();
    } else { //empty-value
      return false;
    }
  } else {
    // field is not part of 'sort' param, but part of 'fl' param
    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
    if (vals.advance(sortDoc.docId) == sortDoc.docId) {
      val = vals.longValue();
    } else {
      return false;
    }
  }
  ew.put(field, val);
  return true;
}
 

@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  final int base = context.docBase;
  final NumericDocValues values = DocValues.getNumeric(context.reader(), "sort_i");
  return new LeafCollector() {
    
    @Override
    public void setScorer(Scorable scorer) throws IOException {}
    
    public void collect(int doc) throws IOException {
      long value;
      if (values.advanceExact(doc)) {
        value = values.longValue();
      } else {
        value = 0;
      }
      list.add(new ScoreDoc(doc+base, (float) value));
    }
  };
}
 

@Benchmark
public LongObjectHashMap<Long> measureGroupingOnNumericDocValues() throws Exception {
    Weight weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    LeafReaderContext leaf = searcher.getTopReaderContext().leaves().get(0);
    Scorer scorer = weight.scorer(leaf);
    NumericDocValues docValues = DocValues.getNumeric(leaf.reader(), "x");
    DocIdSetIterator docIt = scorer.iterator();
    LongObjectHashMap<Long> sumByKey = new LongObjectHashMap<>();
    for (int docId = docIt.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docIt.nextDoc()) {
        if (docValues.advanceExact(docId)) {
            long number = docValues.longValue();
            sumByKey.compute(number, (key, oldValue) -> {
                if (oldValue == null) {
                    return number;
                } else {
                    return oldValue + number;
                }
            });
        }
    }
    return sumByKey;
}
 

@Test
public void testSimilarities() throws IOException {

  MemoryIndex mi = new MemoryIndex();
  mi.addField("f1", "a long text field that contains many many terms", analyzer);

  IndexSearcher searcher = mi.createSearcher();
  LeafReader reader = (LeafReader) searcher.getIndexReader();
  NumericDocValues norms = reader.getNormValues("f1");
  assertEquals(0, norms.nextDoc());
  float n1 = norms.longValue();

  // Norms are re-computed when we change the Similarity
  mi.setSimilarity(new Similarity() {

    @Override
    public long computeNorm(FieldInvertState state) {
      return 74;
    }

    @Override
    public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
      throw new UnsupportedOperationException();
    }

  });
  norms = reader.getNormValues("f1");
  assertEquals(0, norms.nextDoc());
  float n2 = norms.longValue();

  assertTrue(n1 != n2);
  TestUtil.checkReader(reader);
}
 

static DocumentField of(FieldInfo finfo, IndexableField field, IndexReader reader, int docId)
    throws IOException {

  Objects.requireNonNull(finfo);
  Objects.requireNonNull(reader);

  DocumentField dfield = new DocumentField();

  dfield.name = finfo.name;
  dfield.idxOptions = finfo.getIndexOptions();
  dfield.hasTermVectors = finfo.hasVectors();
  dfield.hasPayloads = finfo.hasPayloads();
  dfield.hasNorms = finfo.hasNorms();

  if (finfo.hasNorms()) {
    NumericDocValues norms = MultiDocValues.getNormValues(reader, finfo.name);
    if (norms.advanceExact(docId)) {
      dfield.norm = norms.longValue();
    }
  }

  dfield.dvType = finfo.getDocValuesType();

  dfield.pointDimensionCount = finfo.getPointDimensionCount();
  dfield.pointNumBytes = finfo.getPointNumBytes();

  if (field != null) {
    dfield.isStored = field.fieldType().stored();
    dfield.stringValue = field.stringValue();
    if (field.binaryValue() != null) {
      dfield.binaryValue = BytesRef.deepCopyOf(field.binaryValue());
    }
    dfield.numericValue = field.numericValue();
  }

  return dfield;
}
 

private static LongValues toLongValues(NumericDocValues in) {
  return new LongValues() {
    @Override
    public long longValue() throws IOException {
      return in.longValue();
    }

    @Override
    public boolean advanceExact(int target) throws IOException {
      return in.advanceExact(target);
    }

  };
}
 

private boolean assertDocSoftDeleted(LeafReader leafReader, int segmentDocId) throws IOException {
    final NumericDocValues ndv = leafReader.getNumericDocValues(Lucene.SOFT_DELETES_FIELD);
    if (ndv == null || ndv.advanceExact(segmentDocId) == false) {
        throw new IllegalStateException("DocValues for field [" + Lucene.SOFT_DELETES_FIELD + "] is not found");
    }
    return ndv.longValue() == 1;
}
 

protected ScoreDocKey insert(ScoreDocKey addition, int docBase,
    NumericDocValues keys) throws IOException {
  if ((globalQueue.size() >= numHits)
      && (globalQueue.lessThan(addition, globalQueue.top()))) {
    // Queue is full and proposed addition is not a globally
    // competitive score
    return addition;
  }
  // The addition stands a chance of being entered - check the
  // key-specific restrictions.
  // We delay fetching the key until we are certain the score is globally
  // competitive. We need to adjust the ScoreDoc's global doc value to be
  // a leaf reader value when looking up keys
  int leafDocID = addition.doc - docBase;
  long value;
  if (keys.advanceExact(leafDocID)) {
    value = keys.longValue();
  } else {
    value = 0;
  }
  addition.key = value;

  // For this to work the choice of key class needs to implement
  // hashcode and equals.
  ScoreDocKeyQueue thisKeyQ = perKeyQueues.get(addition.key);

  if (thisKeyQ == null) {
    if (sparePerKeyQueues.size() == 0) {
      thisKeyQ = new ScoreDocKeyQueue(maxNumPerKey);
    } else {
      thisKeyQ = sparePerKeyQueues.pop();
    }
    perKeyQueues.put(addition.key, thisKeyQ);
  }
  ScoreDocKey perKeyOverflow = thisKeyQ.insertWithOverflow(addition);
  if (perKeyOverflow == addition) {
    // This key group has reached capacity and our proposed addition
    // was not competitive in the group - do not insert into the
    // main PQ or the key will be overly-populated in final results.
    return addition;
  }
  if (perKeyOverflow == null) {
    // This proposed addition is also locally competitive within the
    // key group - make a global entry and return
    ScoreDocKey globalOverflow = globalQueue.insertWithOverflow(addition);
    perKeyGroupRemove(globalOverflow);
    return globalOverflow;
  }
  // For the given key, we have reached max capacity but the new addition
  // is better than a prior entry that still exists in the global results
  // - request the weaker-scoring entry to be removed from the global
  // queue.
  globalQueue.remove(perKeyOverflow);
  // Add the locally-competitive addition into the globally queue
  globalQueue.add(addition);
  return perKeyOverflow;
}
 

public void testParentChildrenBlockJoinQuery() throws Exception {
  int numParentDocs = 8 + random().nextInt(8);
  int maxChildDocsPerParent = 8 + random().nextInt(8);

  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  for (int i = 0; i < numParentDocs; i++) {
    int numChildDocs = random().nextInt(maxChildDocsPerParent);
    List<Document> docs = new ArrayList<>(numChildDocs + 1);
    for (int j = 0; j < numChildDocs; j++) {
      Document childDoc = new Document();
      childDoc.add(new StringField("type", "child", Field.Store.NO));
      childDoc.add(new NumericDocValuesField("score", j + 1));
      docs.add(childDoc);
    }

    Document parenDoc = new Document();
    parenDoc.add(new StringField("type", "parent", Field.Store.NO));
    parenDoc.add(new NumericDocValuesField("num_child_docs", numChildDocs));
    docs.add(parenDoc);
    writer.addDocuments(docs);
  }

  IndexReader reader = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(reader);
  BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term("type", "parent")));
  Query childQuery = new BooleanQuery.Builder()
      .add(new TermQuery(new Term("type", "child")), BooleanClause.Occur.FILTER)
      .add(TestJoinUtil.numericDocValuesScoreQuery("score"), BooleanClause.Occur.SHOULD)
      .build();

  TopDocs parentDocs = searcher.search(new TermQuery(new Term("type", "parent")), numParentDocs);
  assertEquals(parentDocs.scoreDocs.length, numParentDocs);
  for (ScoreDoc parentScoreDoc : parentDocs.scoreDocs) {
    LeafReaderContext leafReader = reader.leaves().get(ReaderUtil.subIndex(parentScoreDoc.doc, reader.leaves()));
    NumericDocValues numericDocValuesField = leafReader.reader().getNumericDocValues("num_child_docs");
    numericDocValuesField.advance(parentScoreDoc.doc - leafReader.docBase);
    long expectedChildDocs = numericDocValuesField.longValue();

    ParentChildrenBlockJoinQuery parentChildrenBlockJoinQuery =
        new ParentChildrenBlockJoinQuery(parentFilter, childQuery, parentScoreDoc.doc);
    TopDocs topDocs = searcher.search(parentChildrenBlockJoinQuery, maxChildDocsPerParent);
    assertEquals(expectedChildDocs, topDocs.totalHits.value);
    if (expectedChildDocs > 0) {
      for (int i = 0; i < topDocs.scoreDocs.length; i++) {
        ScoreDoc childScoreDoc = topDocs.scoreDocs[i];
        assertEquals(expectedChildDocs - i, childScoreDoc.score, 0);
      }
    }

  }

  reader.close();
  dir.close();
}
 

/** test random generated lines */
protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // line
    Object queryLine = randomQueryLine(shapes);
    Component2D queryLine2D = toLine2D(queryLine);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), POINT_LINE_RELATIONS);
    Query query = newLineQuery(FIELD_NAME, queryRelation, queryLine);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryLine2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryLine);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated polygons */
protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryPolygon = randomQueryPolygon();
    Component2D queryPoly2D = toPolygon2D(queryPolygon);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newPolygonQuery(FIELD_NAME, queryRelation, queryPolygon);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryPolygon);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated point queries */
protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter+1) + " of " + iters + " s=" + s);
    }

    Object[] queryPoints = nextPoints();
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Component2D queryPoly2D;
    Query query;
    if (queryRelation == QueryRelation.CONTAINS) {
      queryPoly2D = toPoint2D(queryPoints[0]);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints[0]);
    } else {
      queryPoly2D = toPoint2D(queryPoints);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints);
    }

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;

      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  rect=Points(" + Arrays.toString(queryPoints) + ")\n");
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated circles */
protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryCircle = randomQueryCircle();
    Component2D queryCircle2D = toCircle2D(queryCircle);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newDistanceQuery(FIELD_NAME, queryRelation, queryCircle);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryCircle2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  distanceQuery=" + queryCircle.toString());
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}