Java Code Examples for org.apache.lucene.index.RandomIndexWriter#deleteDocuments()

public void testToChildBlockJoinQueryExplain() throws Exception {
  final Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir);

  final List<Document> docs = new ArrayList<>();
  docs.add(makeJob("java", 2007));
  docs.add(makeJob("python", 2010));
  docs.add(makeResume("Lisa", "United Kingdom"));
  w.addDocuments(docs);

  docs.clear();
  docs.add(makeJob("java", 2006));
  docs.add(makeJob("ruby", 2005));
  docs.add(makeResume("Frank", "United States"));
  w.addDocuments(docs);
  w.deleteDocuments(new Term("skill", "java")); // delete the first child of every parent

  IndexReader r = w.getReader();
  w.close();
  IndexSearcher s = newSearcher(r, false);

  // Create a filter that defines "parent" documents in the index - in this case resumes
  BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
  Query parentQuery = new PrefixQuery(new Term("country", "United"));

  ToChildBlockJoinQuery toChildQuery = new ToChildBlockJoinQuery(parentQuery, parentsFilter);

  TopDocs hits = s.search(toChildQuery, 10);
  assertEquals(hits.scoreDocs.length, 2);
  for (int i = 0; i < hits.scoreDocs.length; i++) {
    assertEquals(hits.scoreDocs[i].score, s.explain(toChildQuery, hits.scoreDocs[i].doc).getValue().doubleValue(), 0f);
  }

  r.close();
  dir.close();
}
 

public void testInconsistentDeletes() throws IOException {
  final Directory dir = newDirectory();
  final IndexWriterConfig iwc = newIndexWriterConfig();
  iwc.setMergePolicy(NoMergePolicy.INSTANCE); // so that deletions don't trigger merges
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);

  List<Document> block = new ArrayList<>();
  final int numChildren = TestUtil.nextInt(random(), 1, 3);
  for (int i = 0; i < numChildren; ++i) {
    Document doc = new Document();
    doc.add(new StringField("child", Integer.toString(i), Store.NO));
    block.add(doc);
  }
  Document parent = new Document();
  parent.add(new StringField("parent", "true", Store.NO));
  block.add(parent);
  w.addDocuments(block);

  if (random().nextBoolean()) {
    w.deleteDocuments(new Term("parent", "true"));
  } else {
    // delete any of the children
    w.deleteDocuments(new Term("child", Integer.toString(random().nextInt(numChildren))));
  }

  final IndexReader reader = w.getReader();
  w.close();

  BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("parent", "true")));
  try {
    expectThrows(IllegalStateException.class, () -> CheckJoinIndex.check(reader, parentsFilter));
  } finally {
    reader.close();
    dir.close();
  }
}
 

public void testParentScoringBug() throws Exception {
  final Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir);

  final List<Document> docs = new ArrayList<>();
  docs.add(makeJob("java", 2007));
  docs.add(makeJob("python", 2010));
  docs.add(makeResume("Lisa", "United Kingdom"));
  w.addDocuments(docs);

  docs.clear();
  docs.add(makeJob("java", 2006));
  docs.add(makeJob("ruby", 2005));
  docs.add(makeResume("Frank", "United States"));
  w.addDocuments(docs);
  w.deleteDocuments(new Term("skill", "java")); // delete the first child of every parent

  IndexReader r = w.getReader();
  w.close();
  IndexSearcher s = newSearcher(r, false);

  // Create a filter that defines "parent" documents in the index - in this case resumes
  BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
  Query parentQuery = new PrefixQuery(new Term("country", "United"));

  ToChildBlockJoinQuery toChildQuery = new ToChildBlockJoinQuery(parentQuery, parentsFilter);

  TopDocs hits = s.search(toChildQuery, 10);
  assertEquals(hits.scoreDocs.length, 2);
  for (int i = 0; i < hits.scoreDocs.length; i++) {
    if (hits.scoreDocs[i].score == 0.0)
      fail("Failed to calculate score for hit #"+i);
  }

  r.close();
  dir.close();
}
 

public void testNearestNeighborWithAllDeletedDocs() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, getIndexWriterConfig());
  Document doc = new Document();
  doc.add(new FloatPoint("point", 40.0f, 50.0f));
  doc.add(new StringField("id", "0", Field.Store.YES));
  w.addDocument(doc);
  doc = new Document();
  doc.add(new FloatPoint("point", 45.0f, 55.0f));
  doc.add(new StringField("id", "1", Field.Store.YES));
  w.addDocument(doc);

  DirectoryReader r = w.getReader();
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps with its own points impl:
  IndexSearcher s = newSearcher(r, false);
  FieldDoc hit = (FieldDoc)FloatPointNearestNeighbor.nearest(s, "point", 1, 40.0f, 50.0f).scoreDocs[0];
  assertEquals("0", r.document(hit.doc).getField("id").stringValue());
  r.close();

  w.deleteDocuments(new Term("id", "0"));
  w.deleteDocuments(new Term("id", "1"));
  r = w.getReader();
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps with its own points impl:
  s = newSearcher(r, false);
  assertEquals(0, FloatPointNearestNeighbor.nearest(s, "point", 1, 40.0f, 50.0f).scoreDocs.length);
  r.close();
  w.close();
  dir.close();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  Random random = random();
  directory = newDirectory();
  stopword = "" + randomChar();
  CharacterRunAutomaton stopset = new CharacterRunAutomaton(Automata.makeString(stopword));
  analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false, stopset);
  RandomIndexWriter iw = new RandomIndexWriter(random, directory, analyzer);
  Document doc = new Document();
  Field id = new StringField("id", "", Field.Store.NO);
  Field field = new TextField("field", "", Field.Store.NO);
  doc.add(id);
  doc.add(field);
  
  // index some docs
  int numDocs = TEST_NIGHTLY ? atLeast(1000) : atLeast(100);
  for (int i = 0; i < numDocs; i++) {
    id.setStringValue(Integer.toString(i));
    field.setStringValue(randomFieldContents());
    iw.addDocument(doc);
  }
  
  // delete some docs
  int numDeletes = numDocs/20;
  for (int i = 0; i < numDeletes; i++) {
    Term toDelete = new Term("id", Integer.toString(random.nextInt(numDocs)));
    if (random.nextBoolean()) {
      iw.deleteDocuments(toDelete);
    } else {
      iw.deleteDocuments(new TermQuery(toDelete));
    }
  }
  
  reader = iw.getReader();
  s1 = newSearcher(reader);
  s2 = newSearcher(reader);
  iw.close();
}
 

public void testRandom() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new NumericDocValuesField("dv1", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 2));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new DocValuesFieldExistsQuery("dv1"), false);
    assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new DocValuesFieldExistsQuery("dv2"), false);

    reader.close();
    dir.close();
  }
}
 

public void testScore() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new TextField("text1", "value", Store.NO));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    final float boost = random().nextFloat() * 10;
    final Query ref = new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("has_value", "yes"))), boost);

    final Query q1 = new BoostQuery(new NormsFieldExistsQuery("text1"), boost);
    assertSameMatches(searcher, ref, q1, true);

    reader.close();
    dir.close();
  }
}
 

public void testScore() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new NumericDocValuesField("dv1", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 2));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    final float boost = random().nextFloat() * 10;
    final Query ref = new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("has_value", "yes"))), boost);

    final Query q1 = new BoostQuery(new DocValuesFieldExistsQuery("dv1"), boost);
    assertSameMatches(searcher, ref, q1, true);

    final Query q2 = new BoostQuery(new DocValuesFieldExistsQuery("dv2"), boost);
    assertSameMatches(searcher, ref, q2, true);

    reader.close();
    dir.close();
  }
}
 

public void testApproximation() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new TextField("text1", "value", Store.NO));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    BooleanQuery.Builder ref = new BooleanQuery.Builder();
    ref.add(new TermQuery(new Term("f", "yes")), Occur.MUST);
    ref.add(new TermQuery(new Term("has_value", "yes")), Occur.FILTER);

    BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
    bq1.add(new TermQuery(new Term("f", "yes")), Occur.MUST);
    bq1.add(new NormsFieldExistsQuery("text1"), Occur.FILTER);
    assertSameMatches(searcher, ref.build(), bq1.build(), true);

    reader.close();
    dir.close();
  }
}
 

public void testRandom() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new TextField("text1", "value", Store.NO));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new NormsFieldExistsQuery("text1"), false);

    reader.close();
    dir.close();
  }
}
 

private void doTestSortedVsFieldCache(int minLength, int maxLength) throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
  Document doc = new Document();
  Field idField = new StringField("id", "", Field.Store.NO);
  Field indexedField = new StringField("indexed", "", Field.Store.NO);
  Field dvField = new SortedDocValuesField("dv", new BytesRef());
  doc.add(idField);
  doc.add(indexedField);
  doc.add(dvField);
  
  // index some docs
  int numDocs = atLeast(300);
  for (int i = 0; i < numDocs; i++) {
    idField.setStringValue(Integer.toString(i));
    final int length;
    if (minLength == maxLength) {
      length = minLength; // fixed length
    } else {
      length = TestUtil.nextInt(random(), minLength, maxLength);
    }
    String value = TestUtil.randomSimpleString(random(), length);
    indexedField.setStringValue(value);
    dvField.setBytesValue(new BytesRef(value));
    writer.addDocument(doc);
    if (random().nextInt(31) == 0) {
      writer.commit();
    }
  }
  
  // delete some docs
  int numDeletions = random().nextInt(numDocs/10);
  for (int i = 0; i < numDeletions; i++) {
    int id = random().nextInt(numDocs);
    writer.deleteDocuments(new Term("id", Integer.toString(id)));
  }
  writer.close();
  
  // compare
  DirectoryReader ir = DirectoryReader.open(dir);
  for (LeafReaderContext context : ir.leaves()) {
    LeafReader r = context.reader();
    SortedDocValues expected = FieldCache.DEFAULT.getTermsIndex(r, "indexed");
    SortedDocValues actual = r.getSortedDocValues("dv");
    assertEquals(r.maxDoc(), expected, actual);
  }
  ir.close();
  dir.close();
}
 

public void testDuel() throws IOException {
  final int iters = atLeast(2);
  final String field = "f";
  for (int iter = 0; iter < iters; ++iter) {
    final List<BytesRef> allTerms = new ArrayList<>();
    final int numTerms = TestUtil.nextInt(random(), 1, 1 << TestUtil.nextInt(random(), 1, 10));
    for (int i = 0; i < numTerms; ++i) {
      final String value = TestUtil.randomAnalysisString(random(), 10, true);
      allTerms.add(new BytesRef(value));
    }
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final BytesRef term = allTerms.get(random().nextInt(allTerms.size()));
      doc.add(new StringField(field, term, Store.NO));
      iw.addDocument(doc);
    }
    if (numTerms > 1 && random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term(field, allTerms.get(0))));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    if (reader.numDocs() == 0) {
      // may occasionally happen if all documents got the same term
      IOUtils.close(reader, dir);
      continue;
    }

    for (int i = 0; i < 100; ++i) {
      final float boost = random().nextFloat() * 10;
      final int numQueryTerms = TestUtil.nextInt(random(), 1, 1 << TestUtil.nextInt(random(), 1, 8));
      List<BytesRef> queryTerms = new ArrayList<>();
      for (int j = 0; j < numQueryTerms; ++j) {
        queryTerms.add(allTerms.get(random().nextInt(allTerms.size())));
      }
      final BooleanQuery.Builder bq = new BooleanQuery.Builder();
      for (BytesRef t : queryTerms) {
        bq.add(new TermQuery(new Term(field, t)), Occur.SHOULD);
      }
      final Query q1 = new ConstantScoreQuery(bq.build());
      final Query q2 = new TermInSetQuery(field, queryTerms);
      assertSameMatches(searcher, new BoostQuery(q1, boost), new BoostQuery(q2, boost), true);
    }

    reader.close();
    dir.close();
  }
}
 

private void createRandomIndex(boolean singleSortedSegment) throws IOException {
  dir = newDirectory();
  numDocs = atLeast(150);
  final int numTerms = TestUtil.nextInt(random(), 1, numDocs / 5);
  Set<String> randomTerms = new HashSet<>();
  while (randomTerms.size() < numTerms) {
    randomTerms.add(TestUtil.randomSimpleString(random()));
  }
  terms = new ArrayList<>(randomTerms);
  final long seed = random().nextLong();
  final IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(new Random(seed)));
  if (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {
    // MockRandomMP randomly wraps the leaf readers which makes merging angry
    iwc.setMergePolicy(newTieredMergePolicy());
  }
  iwc.setMergeScheduler(new SerialMergeScheduler()); // for reproducible tests
  iwc.setIndexSort(sort);
  iw = new RandomIndexWriter(new Random(seed), dir, iwc);
  iw.setDoRandomForceMerge(false); // don't do this, it may happen anyway with MockRandomMP
  for (int i = 0; i < numDocs; ++i) {
    final Document doc = randomDocument();
    iw.addDocument(doc);
    if (i == numDocs / 2 || (i != numDocs - 1 && random().nextInt(8) == 0)) {
      iw.commit();
    }
    if (random().nextInt(15) == 0) {
      final String term = RandomPicks.randomFrom(random(), terms);
      iw.deleteDocuments(new Term("s", term));
    }
  }
  if (singleSortedSegment) {
    iw.forceMerge(1);
  }
  else if (random().nextBoolean()) {
    iw.forceMerge(FORCE_MERGE_MAX_SEGMENT_COUNT);
  }
  reader = iw.getReader();
  if (reader.numDocs() == 0) {
    iw.addDocument(new Document());
    reader.close();
    reader = iw.getReader();
  }
}
 

public void testRandom() throws IOException {
  Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  TextField f = new TextField("foo", "foo", Store.NO);
  doc.add(f);
  w.addDocument(doc);
  IndexReader reader = w.getReader();

  final int maxSize;
  final long maxRamBytesUsed;
  final int iters;

  if (TEST_NIGHTLY) {
    maxSize = TestUtil.nextInt(random(), 1, 10000);
    maxRamBytesUsed = TestUtil.nextLong(random(), 1, 5000000);
    iters = atLeast(20000);
  } else {
    maxSize = TestUtil.nextInt(random(), 1, 1000);
    maxRamBytesUsed = TestUtil.nextLong(random(), 1, 500000);
    iters = atLeast(2000);
  }

  final LRUQueryCache queryCache = new LRUQueryCache(maxSize, maxRamBytesUsed, context -> random().nextBoolean(), Float.POSITIVE_INFINITY);
  IndexSearcher uncachedSearcher = null;
  IndexSearcher cachedSearcher = null;

  for (int i = 0; i < iters; ++i) {
    if (i == 0 || random().nextInt(100) == 1) {
      reader.close();
      f.setStringValue(RandomPicks.randomFrom(random(), Arrays.asList("foo", "bar", "bar baz")));
      w.addDocument(doc);
      if (random().nextBoolean()) {
        w.deleteDocuments(buildRandomQuery(0));
      }
      reader = w.getReader();
      uncachedSearcher = newSearcher(reader);
      uncachedSearcher.setQueryCache(null);
      cachedSearcher = newSearcher(reader);
      cachedSearcher.setQueryCache(queryCache);
      cachedSearcher.setQueryCachingPolicy(ALWAYS_CACHE);
    }
    final Query q = buildRandomQuery(0);
    assertEquals(uncachedSearcher.count(q), cachedSearcher.count(q));
    if (rarely()) {
      queryCache.assertConsistent();
    }
  }
  queryCache.assertConsistent();
  w.close();
  reader.close();
  dir.close();
  queryCache.assertConsistent();
}
 

private void doTestDuelPointRangeSortedRangeQuery(boolean sortedSet, int maxValuesPerDoc) throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final int numValues = TestUtil.nextInt(random(), 0, maxValuesPerDoc);
      for (int j = 0; j < numValues; ++j) {
        final long value = TestUtil.nextLong(random(), -100, 10000);
        byte[] encoded = new byte[Long.BYTES];
        LongPoint.encodeDimension(value, encoded, 0);
        if (sortedSet) {
          doc.add(new SortedSetDocValuesField("dv", new BytesRef(encoded)));
        } else {
          doc.add(new SortedDocValuesField("dv", new BytesRef(encoded)));
        }
        doc.add(new LongPoint("idx", value));
      }
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, 10L));
    }
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader, false);
    iw.close();

    for (int i = 0; i < 100; ++i) {
      long min = random().nextBoolean() ? Long.MIN_VALUE : TestUtil.nextLong(random(), -100, 10000);
      long max = random().nextBoolean() ? Long.MAX_VALUE : TestUtil.nextLong(random(), -100, 10000);
      byte[] encodedMin = new byte[Long.BYTES];
      byte[] encodedMax = new byte[Long.BYTES];
      LongPoint.encodeDimension(min, encodedMin, 0);
      LongPoint.encodeDimension(max, encodedMax, 0);
      boolean includeMin = true;
      boolean includeMax = true;
      if (random().nextBoolean()) {
        includeMin = false;
        min++;
      }
      if (random().nextBoolean()) {
        includeMax = false;
        max--;
      }
      final Query q1 = LongPoint.newRangeQuery("idx", min, max);
      final Query q2;
      if (sortedSet) {
        q2 = SortedSetDocValuesField.newSlowRangeQuery("dv",
            min == Long.MIN_VALUE && random().nextBoolean() ? null : new BytesRef(encodedMin),
            max == Long.MAX_VALUE && random().nextBoolean() ? null : new BytesRef(encodedMax),
            includeMin, includeMax);
      } else {
        q2 = SortedDocValuesField.newSlowRangeQuery("dv",
            min == Long.MIN_VALUE && random().nextBoolean() ? null : new BytesRef(encodedMin),
            max == Long.MAX_VALUE && random().nextBoolean() ? null : new BytesRef(encodedMax),
            includeMin, includeMax);
      }
      assertSameMatches(searcher, q1, q2, false);
    }

    reader.close();
    dir.close();
  }
}
 

private void doTestDuelPointRangeNumericRangeQuery(boolean sortedNumeric, int maxValuesPerDoc) throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final int numValues = TestUtil.nextInt(random(), 0, maxValuesPerDoc);
      for (int j = 0; j < numValues; ++j) {
        final long value = TestUtil.nextLong(random(), -100, 10000);
        if (sortedNumeric) {
          doc.add(new SortedNumericDocValuesField("dv", value));
        } else {
          doc.add(new NumericDocValuesField("dv", value));
        }
        doc.add(new LongPoint("idx", value));
      }
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, 10L));
    }
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader, false);
    iw.close();

    for (int i = 0; i < 100; ++i) {
      final long min = random().nextBoolean() ? Long.MIN_VALUE : TestUtil.nextLong(random(), -100, 10000);
      final long max = random().nextBoolean() ? Long.MAX_VALUE : TestUtil.nextLong(random(), -100, 10000);
      final Query q1 = LongPoint.newRangeQuery("idx", min, max);
      final Query q2;
      if (sortedNumeric) {
        q2 = SortedNumericDocValuesField.newSlowRangeQuery("dv", min, max);
      } else {
        q2 = NumericDocValuesField.newSlowRangeQuery("dv", min, max);
      }
      assertSameMatches(searcher, q1, q2, false);
    }

    reader.close();
    dir.close();
  }
}
 

public void testCount() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  final int numDocs = atLeast(100);
  for (int i = 0; i < numDocs; ++i) {
    Document doc = new Document();
    if (random().nextBoolean()) {
      doc.add(new StringField("foo", "bar", Store.NO));
    }
    if (random().nextBoolean()) {
      doc.add(new StringField("foo", "baz", Store.NO));
    }
    if (rarely()) {
      doc.add(new StringField("delete", "yes", Store.NO));
    }
    w.addDocument(doc);
  }
  for (boolean delete : new boolean[] {false, true}) {
    if (delete) {
      w.deleteDocuments(new Term("delete", "yes"));
    }
    final IndexReader reader = w.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    // Test multiple queries, some of them are optimized by IndexSearcher.count()
    for (Query query : Arrays.asList(
        new MatchAllDocsQuery(),
        new MatchNoDocsQuery(),
        new TermQuery(new Term("foo", "bar")),
        new ConstantScoreQuery(new TermQuery(new Term("foo", "baz"))),
        new BooleanQuery.Builder()
          .add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
          .add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD)
          .build()
        )) {
      assertEquals(searcher.count(query), searcher.search(query, 1).totalHits.value);
    }
    reader.close();
  }
  w.close();
  dir.close();
}
 

public void testApproximation() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new NumericDocValuesField("dv1", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 2));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    BooleanQuery.Builder ref = new BooleanQuery.Builder();
    ref.add(new TermQuery(new Term("f", "yes")), Occur.MUST);
    ref.add(new TermQuery(new Term("has_value", "yes")), Occur.FILTER);

    BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
    bq1.add(new TermQuery(new Term("f", "yes")), Occur.MUST);
    bq1.add(new DocValuesFieldExistsQuery("dv1"), Occur.FILTER);
    assertSameMatches(searcher, ref.build(), bq1.build(), true);

    BooleanQuery.Builder bq2 = new BooleanQuery.Builder();
    bq2.add(new TermQuery(new Term("f", "yes")), Occur.MUST);
    bq2.add(new DocValuesFieldExistsQuery("dv2"), Occur.FILTER);
    assertSameMatches(searcher, ref.build(), bq2.build(), true);

    reader.close();
    dir.close();
  }
}
 

public void testSameHitsAsPointRangeQuery() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();

    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    boolean reverse = random().nextBoolean();
    SortField sortField = new SortedNumericSortField("dv", SortField.Type.LONG, reverse);
    sortField.setMissingValue(random().nextLong());
    iwc.setIndexSort(new Sort(sortField));

    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);

    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final int numValues = TestUtil.nextInt(random(), 0, 1);
      for (int j = 0; j < numValues; ++j) {
        final long value = TestUtil.nextLong(random(), -100, 10000);
        doc.add(new SortedNumericDocValuesField("dv", value));
        doc.add(new LongPoint("idx", value));
      }
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, 10L));
    }
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader, false);
    iw.close();

    for (int i = 0; i < 100; ++i) {
      final long min = random().nextBoolean() ? Long.MIN_VALUE : TestUtil.nextLong(random(), -100, 10000);
      final long max = random().nextBoolean() ? Long.MAX_VALUE : TestUtil.nextLong(random(), -100, 10000);
      final Query q1 = LongPoint.newRangeQuery("idx", min, max);
      final Query q2 = createQuery("dv", min, max);
      assertSameHits(searcher, q1, q2, false);
    }

    reader.close();
    dir.close();
  }
}