Java Code Examples for org.apache.lucene.util.TestUtil#checkReader()

/** Tests sorting a single document */
public void testSortOneDocument() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newStringField("value", "foo", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = UninvertingReader.wrap(writer.getReader(),
                   Collections.singletonMap("value", Type.SORTED));
  writer.close();
  
  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortField("value", SortField.Type.STRING));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(1, td.totalHits.value);
  assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value"));
  TestUtil.checkReader(ir);
  ir.close();
  dir.close();
}
 

public void testDocsEnumStart() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  MemoryIndex memory = new MemoryIndex(random().nextBoolean(), false, random().nextInt(50) * 1024 * 1024);
  memory.addField("foo", "bar", analyzer);
  LeafReader reader = (LeafReader) memory.createSearcher().getIndexReader();
  TestUtil.checkReader(reader);
  PostingsEnum disi = TestUtil.docs(random(), reader, "foo", new BytesRef("bar"), null, PostingsEnum.NONE);
  int docid = disi.docID();
  assertEquals(-1, docid);
  assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
  
  // now reuse and check again
  TermsEnum te = reader.terms("foo").iterator();
  assertTrue(te.seekExact(new BytesRef("bar")));
  disi = te.postings(disi, PostingsEnum.NONE);
  docid = disi.docID();
  assertEquals(-1, docid);
  assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
  reader.close();
}
 

/** Tests sorting a single document with scores */
public void testSortOneDocumentWithScores() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newStringField("value", "foo", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = UninvertingReader.wrap(writer.getReader(),
                   Collections.singletonMap("value", Type.SORTED));
  writer.close();
  
  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortField("value", SortField.Type.STRING));

  TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10);
  assertEquals(1, expected.totalHits.value);
  TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), 10, sort, true);
  
  assertEquals(expected.totalHits.value, actual.totalHits.value);
  assertEquals(expected.scoreDocs[0].score, actual.scoreDocs[0].score, 0F);
  TestUtil.checkReader(ir);
  ir.close();
  dir.close();
}
 

protected void assertDVIterate(Directory dir) throws IOException {
  DirectoryReader ir = DirectoryReader.open(dir);
  TestUtil.checkReader(ir);
  for (LeafReaderContext context : ir.leaves()) {
    LeafReader r = context.reader();
    NumericDocValues docValues = DocValues.getNumeric(r, "dv");
    docValues.nextDoc();
    for (int i = 0; i < r.maxDoc(); i++) {
      String storedValue = r.document(i).get("stored");
      if (storedValue == null) {
        assertTrue(docValues.docID() > i);
      } else {
        assertEquals(i, docValues.docID());
        assertEquals(Long.parseLong(storedValue), docValues.longValue());
        docValues.nextDoc();
      }
    }
    assertEquals(DocIdSetIterator.NO_MORE_DOCS, docValues.docID());
  }
  ir.close();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  NUM_DOCS = atLeast(500);
  directory = newDirectory();
  RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
  long theLong = Long.MAX_VALUE;
  double theDouble = Double.MAX_VALUE;
  int theInt = Integer.MAX_VALUE;
  float theFloat = Float.MAX_VALUE;
  if (VERBOSE) {
    System.out.println("TEST: setUp");
  }
  for (int i = 0; i < NUM_DOCS; i++){
    Document doc = new Document();
    doc.add(new LegacyLongField("theLong", theLong--, Field.Store.NO));
    doc.add(new LegacyDoubleField("theDouble", theDouble--, Field.Store.NO));
    doc.add(new LegacyIntField("theInt", theInt--, Field.Store.NO));
    doc.add(new LegacyFloatField("theFloat", theFloat--, Field.Store.NO));
    if (i%2 == 0) {
      doc.add(new LegacyIntField("sparse", i, Field.Store.NO));
    }

    if (i%2 == 0) {
      doc.add(new LegacyIntField("numInt", i, Field.Store.NO));
    }
    writer.addDocument(doc);
  }
  IndexReader r = writer.getReader();
  reader = SlowCompositeReaderWrapper.wrap(r);
  TestUtil.checkReader(reader);
  writer.close();
}
 

public void testSortedSetEmptyIndex() throws IOException {
  final Directory dir = newDirectory();
  final IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  iw.close();
  
  final Map<String,Type> UNINVERT_MAP = new LinkedHashMap<String,Type>();
  for (Type t : EnumSet.allOf(Type.class)) {
    UNINVERT_MAP.put(t.name(), t);
  }

  final DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), UNINVERT_MAP);
  TestUtil.checkReader(ir);
  
  final LeafReader composite = SlowCompositeReaderWrapper.wrap(ir);
  TestUtil.checkReader(composite);
  
  for (String f : UNINVERT_MAP.keySet()) { 
    // check the leaves
    // (normally there are none for an empty index, so this is really just future
    // proofing in case that changes for some reason)
    for (LeafReaderContext rc : ir.leaves()) {
      final LeafReader ar = rc.reader();
      assertNull(f + ": Expected no doc values from empty index (leaf)",
                 ar.getSortedSetDocValues(f));
    }
    
    // check the composite
    assertNull(f + ": Expected no doc values from empty index (composite)",
               composite.getSortedSetDocValues(f));
    
  }

  ir.close();
  dir.close();
}
 

public void testCrazyReaderEquals() throws IOException {
  int numdocs = atLeast(20);
  createRandomIndex(numdocs, leftWriter, seed);
  createRandomIndex(numdocs, rightWriter, seed);

  leftReader = wrapReader(leftWriter.getReader());
  rightReader = wrapReader(rightWriter.getReader());
  
  // check that our readers are valid
  TestUtil.checkReader(leftReader);
  TestUtil.checkReader(rightReader);
  
  assertReaderEquals(info, leftReader, rightReader);
}
 

public void test() throws Exception {
  BaseDirectoryWrapper d = newDirectory();
  d.setCheckIndexOnClose(false);
  // we nuke files, but verify the reader still works
  RandomIndexWriter w = new RandomIndexWriter(random(), d);
  int numDocs = atLeast(100);
  for(int i=0;i<numDocs;i++) {
    Document doc = new Document();
    doc.add(newField("foo", "bar", TextField.TYPE_NOT_STORED));
    doc.add(new IntPoint("doc", i));
    doc.add(new IntPoint("doc2d", i, i));
    doc.add(new NumericDocValuesField("dv", i));
    w.addDocument(doc);
  }

  IndexReader r = w.getReader();
  w.commit();
  w.close();

  for (String name : d.listAll()) {
    d.deleteFile(name);
  }

  for(LeafReaderContext cxt : r.leaves()) {
    TestUtil.checkReader(cxt.reader());
  }

  r.close();
  d.close();
}
 

public void testPassesIfWrapped() throws IOException {
  RegexpQuery regex = new RegexpQuery(new Term("field", "worl."));
  SpanQuery wrappedquery = new SpanOrQuery(new SpanMultiTermQueryWrapper<>(regex));

  MemoryIndex mindex = randomMemoryIndex();
  mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there"));

  // This passes though
  assertEquals(0, mindex.search(wrappedquery), 0.00001f);
  TestUtil.checkReader(mindex.createSearcher().getIndexReader());
}
 

public void testNullPointerException() throws IOException {
  RegexpQuery regex = new RegexpQuery(new Term("field", "worl."));
  SpanQuery wrappedquery = new SpanMultiTermQueryWrapper<>(regex);
      
  MemoryIndex mindex = randomMemoryIndex();
  mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there"));

  // This throws an NPE
  assertEquals(0, mindex.search(wrappedquery), 0.00001f);
  TestUtil.checkReader(mindex.createSearcher().getIndexReader());
}
 

public void testDocsAndPositionsEnumStart() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  int numIters = atLeast(3);
  MemoryIndex memory = new MemoryIndex(true, false, random().nextInt(50) * 1024 * 1024);
  for (int i = 0; i < numIters; i++) { // check reuse
    memory.addField("foo", "bar", analyzer);
    LeafReader reader = (LeafReader) memory.createSearcher().getIndexReader();
    TestUtil.checkReader(reader);
    assertEquals(1, reader.terms("foo").getSumTotalTermFreq());
    PostingsEnum disi = reader.postings(new Term("foo", "bar"), PostingsEnum.ALL);
    int docid = disi.docID();
    assertEquals(-1, docid);
    assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
    assertEquals(0, disi.nextPosition());
    assertEquals(0, disi.startOffset());
    assertEquals(3, disi.endOffset());
    
    // now reuse and check again
    TermsEnum te = reader.terms("foo").iterator();
    assertTrue(te.seekExact(new BytesRef("bar")));
    disi = te.postings(disi);
    docid = disi.docID();
    assertEquals(-1, docid);
    assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
    reader.close();
    memory.reset();
  }
}
 

@Test
public void testSimilarities() throws IOException {

  MemoryIndex mi = new MemoryIndex();
  mi.addField("f1", "a long text field that contains many many terms", analyzer);

  IndexSearcher searcher = mi.createSearcher();
  LeafReader reader = (LeafReader) searcher.getIndexReader();
  NumericDocValues norms = reader.getNormValues("f1");
  assertEquals(0, norms.nextDoc());
  float n1 = norms.longValue();

  // Norms are re-computed when we change the Similarity
  mi.setSimilarity(new Similarity() {

    @Override
    public long computeNorm(FieldInvertState state) {
      return 74;
    }

    @Override
    public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
      throw new UnsupportedOperationException();
    }

  });
  norms = reader.getNormValues("f1");
  assertEquals(0, norms.nextDoc());
  float n2 = norms.longValue();

  assertTrue(n1 != n2);
  TestUtil.checkReader(reader);
}
 

@AfterClass
public static void afterClass() throws Exception {
  searcher = null;
  if (null != reader) {
    TestUtil.checkReader(reader);
    reader.close();
    reader = null;
  }
  if (null != directory) {
    directory.close();
    directory = null;
  }
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  NUM_DOCS = atLeast(500);
  NUM_ORDS = atLeast(2);
  directory = newDirectory();
  IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new LogDocMergePolicy()));
  long theLong = Long.MAX_VALUE;
  double theDouble = Double.MAX_VALUE;
  int theInt = Integer.MAX_VALUE;
  float theFloat = Float.MAX_VALUE;
  unicodeStrings = new String[NUM_DOCS];
  multiValued = new BytesRef[NUM_DOCS][NUM_ORDS];
  if (VERBOSE) {
    System.out.println("TEST: setUp");
  }
  for (int i = 0; i < NUM_DOCS; i++){
    Document doc = new Document();
    doc.add(new LongPoint("theLong", theLong--));
    doc.add(new DoublePoint("theDouble", theDouble--));
    doc.add(new IntPoint("theInt", theInt--));
    doc.add(new FloatPoint("theFloat", theFloat--));
    if (i%2 == 0) {
      doc.add(new IntPoint("sparse", i));
    }

    if (i%2 == 0) {
      doc.add(new IntPoint("numInt", i));
    }

    // sometimes skip the field:
    if (random().nextInt(40) != 17) {
      unicodeStrings[i] = generateString(i);
      doc.add(newStringField("theRandomUnicodeString", unicodeStrings[i], Field.Store.YES));
    }

    // sometimes skip the field:
    if (random().nextInt(10) != 8) {
      for (int j = 0; j < NUM_ORDS; j++) {
        String newValue = generateString(i);
        multiValued[i][j] = new BytesRef(newValue);
        doc.add(newStringField("theRandomUnicodeMultiValuedField", newValue, Field.Store.YES));
      }
      Arrays.sort(multiValued[i]);
    }
    writer.addDocument(doc);
  }
  writer.forceMerge(1); // this test relies on one segment and docid order
  IndexReader r = DirectoryReader.open(writer);
  assertEquals(1, r.leaves().size());
  reader = r.leaves().get(0).reader();
  TestUtil.checkReader(reader);
  writer.close();
}
 

public void testRandom() throws Exception {
  Directory dir = newDirectory();

  final int NUM_TERMS = atLeast(20);
  final Set<BytesRef> terms = new HashSet<>();
  while(terms.size() < NUM_TERMS) {
    final String s = TestUtil.randomRealisticUnicodeString(random());
    //final String s = _TestUtil.randomSimpleString(random);
    if (s.length() > 0) {
      terms.add(new BytesRef(s));
    }
  }
  final BytesRef[] termsArray = terms.toArray(new BytesRef[terms.size()]);
  Arrays.sort(termsArray);
  
  final int NUM_DOCS = atLeast(100);

  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));

  // Sometimes swap in codec that impls ord():
  if (random().nextInt(10) == 7) {
    // Make sure terms index has ords:
    Codec codec = TestUtil.alwaysPostingsFormat(TestUtil.getPostingsFormatWithOrds(random()));
    conf.setCodec(codec);
  }
  
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir, conf);

  final int[][] idToOrds = new int[NUM_DOCS][];
  final Set<Integer> ordsForDocSet = new HashSet<>();

  for(int id=0;id<NUM_DOCS;id++) {
    Document doc = new Document();

    doc.add(new LegacyIntField("id", id, Field.Store.YES));
    
    final int termCount = TestUtil.nextInt(random(), 0, 20 * RANDOM_MULTIPLIER);
    while(ordsForDocSet.size() < termCount) {
      ordsForDocSet.add(random().nextInt(termsArray.length));
    }
    final int[] ordsForDoc = new int[termCount];
    int upto = 0;
    if (VERBOSE) {
      System.out.println("TEST: doc id=" + id);
    }
    for(int ord : ordsForDocSet) {
      ordsForDoc[upto++] = ord;
      Field field = newStringField("field", termsArray[ord].utf8ToString(), Field.Store.NO);
      if (VERBOSE) {
        System.out.println("  f=" + termsArray[ord].utf8ToString());
      }
      doc.add(field);
    }
    ordsForDocSet.clear();
    Arrays.sort(ordsForDoc);
    idToOrds[id] = ordsForDoc;
    w.addDocument(doc);
  }
  
  final DirectoryReader r = w.getReader();
  w.close();

  if (VERBOSE) {
    System.out.println("TEST: reader=" + r);
  }

  for(LeafReaderContext ctx : r.leaves()) {
    if (VERBOSE) {
      System.out.println("\nTEST: sub=" + ctx.reader());
    }
    verify(ctx.reader(), idToOrds, termsArray, null);
  }

  // Also test top-level reader: its enum does not support
  // ord, so this forces the OrdWrapper to run:
  if (VERBOSE) {
    System.out.println("TEST: top reader");
  }
  LeafReader slowR = SlowCompositeReaderWrapper.wrap(r);
  TestUtil.checkReader(slowR);
  verify(slowR, idToOrds, termsArray, null);

  FieldCache.DEFAULT.purgeByCacheKey(slowR.getCoreCacheHelper().getKey());

  r.close();
  dir.close();
}
 

public void testFieldInfos() throws IOException {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));

  Document doc = new Document();
  BytesRef idBytes = new BytesRef("id");
  doc.add(new StringField("id", idBytes, Store.YES));
  doc.add(new LegacyIntField("int", 5, Store.YES));
  doc.add(new NumericDocValuesField("dv", 5));
  doc.add(new IntPoint("dint", 5));
  doc.add(new StoredField("stored", 5)); // not indexed
  iw.addDocument(doc);

  iw.forceMerge(1);
  iw.close();

  Map<String, Type> uninvertingMap = new HashMap<>();
  uninvertingMap.put("int", Type.LEGACY_INTEGER);
  uninvertingMap.put("dv", Type.LEGACY_INTEGER);
  uninvertingMap.put("dint", Type.INTEGER_POINT);

  DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), uninvertingMap);
  LeafReader leafReader = ir.leaves().get(0).reader();
  FieldInfos fieldInfos = leafReader.getFieldInfos();
  LeafReader originalLeafReader = ((UninvertingReader)leafReader).getDelegate();

  assertNotSame(originalLeafReader.getFieldInfos(), fieldInfos);
  assertSame("do not rebuild FieldInfo for unaffected fields",
      originalLeafReader.getFieldInfos().fieldInfo("id"), fieldInfos.fieldInfo("id"));

  FieldInfo intFInfo = fieldInfos.fieldInfo("int");
  assertEquals(DocValuesType.NUMERIC, intFInfo.getDocValuesType());
  assertEquals(0, intFInfo.getPointDimensionCount());
  assertEquals(0, intFInfo.getPointIndexDimensionCount());
  assertEquals(0, intFInfo.getPointNumBytes());

  FieldInfo dintFInfo = fieldInfos.fieldInfo("dint");
  assertEquals(DocValuesType.NUMERIC, dintFInfo.getDocValuesType());
  assertEquals(1, dintFInfo.getPointDimensionCount());
  assertEquals(1, dintFInfo.getPointIndexDimensionCount());
  assertEquals(4, dintFInfo.getPointNumBytes());

  FieldInfo dvFInfo = fieldInfos.fieldInfo("dv");
  assertEquals(DocValuesType.NUMERIC, dvFInfo.getDocValuesType());

  FieldInfo storedFInfo = fieldInfos.fieldInfo("stored");
  assertEquals(DocValuesType.NONE, storedFInfo.getDocValuesType());

  TestUtil.checkReader(ir);
  ir.close();
  dir.close();
}
 

/** Tests dv against stored fields with threads (binary/numeric/sorted, no missing) */
public void testThreads() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
  Document doc = new Document();
  Field idField = new StringField("id", "", Field.Store.NO);
  Field storedBinField = new StoredField("storedBin", new byte[0]);
  Field dvBinField = new BinaryDocValuesField("dvBin", new BytesRef());
  Field dvSortedField = new SortedDocValuesField("dvSorted", new BytesRef());
  Field storedNumericField = new StoredField("storedNum", "");
  Field dvNumericField = new NumericDocValuesField("dvNum", 0);
  doc.add(idField);
  doc.add(storedBinField);
  doc.add(dvBinField);
  doc.add(dvSortedField);
  doc.add(storedNumericField);
  doc.add(dvNumericField);
  
  // index some docs
  int numDocs = atLeast(300);
  for (int i = 0; i < numDocs; i++) {
    idField.setStringValue(Integer.toString(i));
    int length = TestUtil.nextInt(random(), 0, 8);
    byte buffer[] = new byte[length];
    random().nextBytes(buffer);
    storedBinField.setBytesValue(buffer);
    dvBinField.setBytesValue(buffer);
    dvSortedField.setBytesValue(buffer);
    long numericValue = random().nextLong();
    storedNumericField.setStringValue(Long.toString(numericValue));
    dvNumericField.setLongValue(numericValue);
    writer.addDocument(doc);
    if (random().nextInt(31) == 0) {
      writer.commit();
    }
  }
  
  // delete some docs
  int numDeletions = random().nextInt(numDocs/10);
  for (int i = 0; i < numDeletions; i++) {
    int id = random().nextInt(numDocs);
    writer.deleteDocuments(new Term("id", Integer.toString(id)));
  }
  writer.close();
  
  // compare
  final DirectoryReader ir = DirectoryReader.open(dir);
  int numThreads = TestUtil.nextInt(random(), 2, 7);
  Thread threads[] = new Thread[numThreads];
  final CountDownLatch startingGun = new CountDownLatch(1);
  
  for (int i = 0; i < threads.length; i++) {
    threads[i] = new Thread() {
      @Override
      public void run() {
        try {
          startingGun.await();
          for (LeafReaderContext context : ir.leaves()) {
            LeafReader r = context.reader();
            BinaryDocValues binaries = r.getBinaryDocValues("dvBin");
            SortedDocValues sorted = r.getSortedDocValues("dvSorted");
            NumericDocValues numerics = r.getNumericDocValues("dvNum");
            for (int j = 0; j < r.maxDoc(); j++) {
              BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
              assertEquals(j, binaries.nextDoc());
              BytesRef scratch = binaries.binaryValue();
              assertEquals(binaryValue, scratch);
              assertEquals(j, sorted.nextDoc());
              scratch = sorted.binaryValue();
              assertEquals(binaryValue, scratch);
              String expected = r.document(j).get("storedNum");
              assertEquals(j, numerics.nextDoc());
              assertEquals(Long.parseLong(expected), numerics.longValue());
            }
          }
          TestUtil.checkReader(ir);
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      }
    };
    threads[i].start();
  }
  startingGun.countDown();
  for (Thread t : threads) {
    t.join();
  }
  ir.close();
  dir.close();
}
 

private void doTestSortedNumericBlocksOfVariousBitsPerValue(LongSupplier counts) throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  conf.setMaxBufferedDocs(atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE));
  conf.setRAMBufferSizeMB(-1);
  conf.setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  final int numDocs = atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE*3);
  final LongSupplier values = blocksOfVariousBPV();
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    
    int valueCount = (int) counts.getAsLong();
    long valueArray[] = new long[valueCount];
    for (int j = 0; j < valueCount; j++) {
      long value = values.getAsLong();
      valueArray[j] = value;
      doc.add(new SortedNumericDocValuesField("dv", value));
    }
    Arrays.sort(valueArray);
    for (int j = 0; j < valueCount; j++) {
      doc.add(new StoredField("stored", Long.toString(valueArray[j])));
    }
    writer.addDocument(doc);
    if (random().nextInt(31) == 0) {
      writer.commit();
    }
  }
  writer.forceMerge(1);

  writer.close();
  
  // compare
  DirectoryReader ir = DirectoryReader.open(dir);
  TestUtil.checkReader(ir);
  for (LeafReaderContext context : ir.leaves()) {
    LeafReader r = context.reader();
    SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");
    for (int i = 0; i < r.maxDoc(); i++) {
      if (i > docValues.docID()) {
        docValues.nextDoc();
      }
      String expected[] = r.document(i).getValues("stored");
      if (i < docValues.docID()) {
        assertEquals(0, expected.length);
      } else {
        String actual[] = new String[docValues.docValueCount()];
        for (int j = 0; j < actual.length; j++) {
          actual[j] = Long.toString(docValues.nextValue());
        }
        assertArrayEquals(expected, actual);
      }
    }
  }
  ir.close();
  dir.close();
}
 

/**
 * Build a randomish document for both Directory and MemoryIndex,
 * and run all the queries against it.
 */
public void assertAgainstDirectory(MemoryIndex memory) throws Exception {
  memory.reset();
  StringBuilder fooField = new StringBuilder();
  StringBuilder termField = new StringBuilder();
 
  // add up to 250 terms to field "foo"
  final int numFooTerms = random().nextInt(250 * RANDOM_MULTIPLIER);
  for (int i = 0; i < numFooTerms; i++) {
    fooField.append(" ");
    fooField.append(randomTerm());
  }

  // add up to 250 terms to field "term"
  final int numTermTerms = random().nextInt(250 * RANDOM_MULTIPLIER);
  for (int i = 0; i < numTermTerms; i++) {
    termField.append(" ");
    termField.append(randomTerm());
  }
  
  Directory dir = new ByteBuffersDirectory();
  Analyzer analyzer = randomAnalyzer();
  IndexWriter writer = new IndexWriter(dir,
                                       new IndexWriterConfig(analyzer).setCodec(
                                           TestUtil.alwaysPostingsFormat(TestUtil.getDefaultPostingsFormat())));
  Document doc = new Document();
  Field field1 = newTextField("foo", fooField.toString(), Field.Store.NO);
  Field field2 = newTextField("term", termField.toString(), Field.Store.NO);
  doc.add(field1);
  doc.add(field2);
  writer.addDocument(doc);
  writer.close();
  
  memory.addField("foo", fooField.toString(), analyzer);
  memory.addField("term", termField.toString(), analyzer);
  
  LeafReader reader = (LeafReader) memory.createSearcher().getIndexReader();
  TestUtil.checkReader(reader);
  DirectoryReader competitor = DirectoryReader.open(dir);
  duellReaders(competitor, reader);
  IOUtils.close(reader, competitor);
  assertAllQueries(memory, dir, analyzer);
  dir.close();
}
 

@SuppressWarnings({"ConstantConditions", "PointlessBooleanExpression"})
@Nightly
public void testTriggerUnInvertLimit() throws IOException {
  final boolean SHOULD_TRIGGER = false; // Set this to true to use the test with the old implementation

  // Ensure enough terms inside of a single UnInvert-pass-structure to trigger the limit
  final int REF_LIMIT = (int) Math.pow(2, 24); // Maximum number of references within a single pass-structure
  final int DOCS = (1<<16)-1;                  // The number of documents within a single pass (simplified)
  final int TERMS = REF_LIMIT/DOCS;            // Each document must have this many references aka terms hit limit

  // disk based Directory and IWC settings to reduce risk of OOM
  Directory dir = newFSDirectory(createTempDir("TestDocTermOrdsUninvertLimit"));
  final IndexWriter w = new IndexWriter(dir,
                                        new IndexWriterConfig(new MockAnalyzer(random()))
                                        .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
                                        .setRAMBufferSizeMB(256.0)
                                        .setMergeScheduler(new ConcurrentMergeScheduler())
                                        .setMergePolicy(newLogMergePolicy(false, 10))
                                        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
                                        .setCodec(TestUtil.getDefaultCodec()));
  
  Document doc = new Document();
  Field field = newTextField("field", "", Field.Store.NO);
  doc.add(field);

  StringBuilder sb = new StringBuilder(TERMS*(Integer.toString(TERMS).length()+1));
  for (int i = 0 ; i < TERMS ; i++) {
    sb.append(" ").append(Integer.toString(i));
  }
  field.setStringValue(sb.toString());

  for (int i = 0 ; i < DOCS ; i++) {
    w.addDocument(doc);
  }
  //System.out.println("\n Finished adding " + DOCS + " documents of " + TERMS + " unique terms");
  w.close();
  
  final IndexReader r = DirectoryReader.open(dir);
  try {
    final LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
    TestUtil.checkReader(ar);
    final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field"); // bigTerms turned off
    if (SHOULD_TRIGGER) {
      fail("DocTermOrds should have failed with a \"Too many values for UnInvertedField\" message");
    }
  } catch (IllegalStateException e) {
    if (!SHOULD_TRIGGER) {
      fail("DocTermsOrd should not have failed with this implementation, but got exception " +
          e.getClass().getSimpleName() + " with message " + e.getMessage());
    }
    // This is (hopefully) "Too many values for UnInvertedField faceting on field field", so all is as expected
  } finally {
    r.close();
    dir.close();
  }
}