Java Code Examples for org.apache.lucene.util.IOUtils#close()

@Test
public void testNoCommit() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  // should fail when there are no commits to snapshot
  expectThrows(IllegalStateException.class, () -> {
    new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
  });

  indexWriter.close();
  IOUtils.close(taxoWriter, taxoDir, indexDir);
}
 

@Override
public FieldsProducer fieldsProducer(SegmentReadState readState)
  throws IOException {

  // Load our ID:
  final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION);
  IndexInput in = readState.directory.openInput(idFileName, readState.context);
  boolean success = false;
  final int id;
  try {
    CodecUtil.checkHeader(in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST);
    id = in.readVInt();
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(in);
    } else {
      IOUtils.close(in);
    }
  }
  
  synchronized(state) {
    return state.get(id);
  }
}
 

private void checkTokens(Token[] tokens) throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
  boolean success = false;
  try {
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    // store some term vectors for the checkindex cross-check
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorOffsets(true);
   
    Document doc = new Document();
    doc.add(new Field("body", new CannedTokenStream(tokens), ft));
    riw.addDocument(doc);
    riw.close();
    success = true;
  } finally {
    if (success) {
      IOUtils.close(dir);
    } else {
      IOUtils.closeWhileHandlingException(riw, dir);
    }
  }
}
 

public void testFullMergeAddIndexesDir() throws Exception {
  Directory[] input = new Directory[] { newDirectory(), newDirectory() };
  Directory target = newDirectory();
  
  for (Directory dir : input) {
    createDir(dir);
  }
  
  IndexWriter writer = createWriter(target);
  writer.addIndexes(input);
  writer.forceMerge(1);
  writer.close();

  verifyIndex(target);

  IOUtils.close(target, input[0], input[1]);
}
 

@Override
public void close() throws IOException {
  boolean success = false;
  try {
    if (meta != null) {
      meta.writeInt(-1); // write EOF marker
      CodecUtil.writeFooter(meta); // write checksum
    }
    if (data != null) {
      CodecUtil.writeFooter(data); // write checksum
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(data, meta);
    } else {
      IOUtils.closeWhileHandlingException(data, meta);
    }
    meta = data = null;
  }
}
 

public void testDetectHierarchicalField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path", "other"));
  expectThrows(IllegalArgumentException.class, () -> {
    config.build(taxoWriter, doc);
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 

@Test
public void testEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());
  
  IOUtils.close(ir, analyzer, dir);
}
 

public void testTooManyExpansions() throws Exception {

    final Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName) {
        return new TokenStreamComponents(r -> {}, new CannedTokenStream(
            new Token("a", 0, 1),
            new Token("b", 0, 0, 1)));
      }
    };

    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, 1, true);
    suggester.build(new InputArrayIterator(new Input[] {new Input("a", 1)}));
    assertEquals("[a/1]", suggester.lookup("a", false, 1).toString());
    IOUtils.close(a, tempDir);
  }
 

public Location writeToLocal(BytesReference data) throws IOException {
    final long position;
    final long generation;
    try (ReleasableLock lock = writeLock.acquire()) {
        ensureOpen();
        if (writtenOffset > TRANSLOG_ROLLING_SIZE_BYTES) {
            IOUtils.close(writeChannel);
            tmpTranslogGeneration.incrementAndGet();
            writeChannel = FileChannel.open(this.translogPath.resolve(getFileNameFromId(tmpTranslogGeneration.get())), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
            writtenOffset = 0;
        }
        generation = tmpTranslogGeneration.get();
        position = writtenOffset;
        try {
            data.writeTo(writeChannel);
        } catch (Throwable e) {
            throw e;
        }
        writtenOffset = writtenOffset + data.length();
    }
    return new Translog.Location(generation, position, data.length());
}
 

@Override
public void close() throws IOException {
  try {
    IOUtils.close(fieldsStream, indexWriter, compressor);
  } finally {
    fieldsStream = null;
    indexWriter = null;
    compressor = null;
  }
}
 

public void testBoundingBoxQueriesEquivalence() throws Exception {
  int numShapes = atLeast(20);

  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);

  for (int  i =0; i < numShapes; i++) {
    indexRandomShapes(w.w, nextShape());
  }
  if (random().nextBoolean()) {
    w.forceMerge(1);
  }

  ///// search //////
  IndexReader reader = w.getReader();
  w.close();
  IndexSearcher searcher = newSearcher(reader);

  Rectangle box = GeoTestUtil.nextBox();

  Query q1 = LatLonShape.newBoxQuery(FIELD_NAME, QueryRelation.INTERSECTS, box.minLat, box.maxLat, box.minLon, box.maxLon);
  Query q2 = new LatLonShapeQuery(FIELD_NAME, QueryRelation.INTERSECTS, box);
  assertEquals(searcher.count(q1), searcher.count(q2));
  q1 = LatLonShape.newBoxQuery(FIELD_NAME, QueryRelation.WITHIN, box.minLat, box.maxLat, box.minLon, box.maxLon);
  q2 = new LatLonShapeQuery(FIELD_NAME, QueryRelation.WITHIN, box);
  assertEquals(searcher.count(q1), searcher.count(q2));
  q1 = LatLonShape.newBoxQuery(FIELD_NAME, QueryRelation.CONTAINS, box.minLat, box.maxLat, box.minLon, box.maxLon);
  if (box.crossesDateline()) {
    q2 = LatLonShape.newGeometryQuery(FIELD_NAME, QueryRelation.CONTAINS, box);
  } else {
    q2 = new LatLonShapeQuery(FIELD_NAME, QueryRelation.CONTAINS, box);
  }
  assertEquals(searcher.count(q1), searcher.count(q2));
  q1 = LatLonShape.newBoxQuery(FIELD_NAME, QueryRelation.DISJOINT, box.minLat, box.maxLat, box.minLon, box.maxLon);
  q2 = new LatLonShapeQuery(FIELD_NAME, QueryRelation.DISJOINT, box);
  assertEquals(searcher.count(q1), searcher.count(q2));

  IOUtils.close(w, reader, dir);
}
 

/** blast some random large strings through the analyzer */
public void testRandomHugeStrings() throws Exception {
  Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(),
      new StopAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET), new UnicodeWhitespaceAnalyzer() };
  for (Analyzer analyzer : analyzers) {
    checkRandomData(random(), analyzer, 10*RANDOM_MULTIPLIER, 8192);
  }
  IOUtils.close(analyzers);
}
 

@Override
public void close() throws IOException {
  message("top: now close");

  synchronized (this) {
    state = "closing";
    if (curNRTCopy != null) {
      curNRTCopy.cancel("closing", null);
    }
  }

  synchronized (this) {
    message("top: close mgr");
    mgr.close();

    message("top: decRef lastNRTFiles=" + lastNRTFiles);
    deleter.decRef(lastNRTFiles);
    lastNRTFiles.clear();

    // NOTE: do not decRef these!
    lastCommitFiles.clear();

    message("top: delete if no ref pendingMergeFiles=" + pendingMergeFiles);
    for(String fileName : pendingMergeFiles) {
      deleter.deleteIfNoRef(fileName);
    }
    pendingMergeFiles.clear();
  
    message("top: close dir");
    IOUtils.close(writeFileLock, dir);
  }
  message("top: done close");
  state = "closed";
}
 

public void testIntraSliceDocIDOrder() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  w.addDocument(new Document());
  w.addDocument(new Document());
  w.commit();
  w.addDocument(new Document());
  w.addDocument(new Document());
  w.commit();
  IndexReader r = w.getReader();
  w.close();

  ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
      new LinkedBlockingQueue<Runnable>(),
      new NamedThreadFactory("TestSegmentToThreadMapping"));
  IndexSearcher s = new IndexSearcher(r, service);
  Query query = new MatchAllDocsQuery();

  s.search(query, Integer.MAX_VALUE);

  IndexSearcher.LeafSlice[] slices = s.getSlices();
  assertNotNull(slices);

  for (IndexSearcher.LeafSlice leafSlice : slices) {
    LeafReaderContext[] leafReaderContexts = leafSlice.leaves;
    int previousDocBase = leafReaderContexts[0].docBase;

    for (LeafReaderContext leafReaderContext : leafReaderContexts) {
      assertTrue(previousDocBase <= leafReaderContext.docBase);
      previousDocBase = leafReaderContext.docBase;
    }
  }

  service.shutdown();
  IOUtils.close(r, dir);
}
 

@Override
public void tearDown() throws Exception {
  IOUtils.close(a, b);
  super.tearDown();
}
 

@Test
public void testTwoCustomsSameField() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);

  FacetsConfig config = getConfig();
  config.setIndexFieldName("Band", "$music");
  config.setIndexFieldName("Composer", "$music");
  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  Map<String,Facets> facetsMap = new HashMap<>();
  Facets facets2 = getTaxonomyFacetCounts(tr, config, sfc, "$music");
  facetsMap.put("Band", facets2);
  facetsMap.put("Composer", facets2);
  Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));

  // Obtain facets results and hand-test them
  assertCorrectResults(facets);

  assertOrdinalsExist("$facets", ir);
  assertOrdinalsExist("$music", ir);
  assertOrdinalsExist("$music", ir);

  iw.close();
  IOUtils.close(tr, ir, tw, indexDir, taxoDir);
}
 

public void testIterableThrowsException() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  int iters = atLeast(100);
  int docCount = 0;
  int docId = 0;
  Set<String> liveIds = new HashSet<>();
  for (int i = 0; i < iters; i++) {
    int numDocs = atLeast(4);
    for (int j = 0; j < numDocs; j++) {
      String id = Integer.toString(docId++);
      final List<IndexableField> fields = new ArrayList<>();
      fields.add(new StringField("id", id, Field.Store.YES));
      fields.add(new StringField("foo", TestUtil.randomSimpleString(random()), Field.Store.NO));
      docId++;

      boolean success = false;
      try {
        w.addDocument(new RandomFailingIterable<IndexableField>(fields, random()));
        success = true;
      } catch (RuntimeException e) {
        assertEquals("boom", e.getMessage());
      } finally {
        if (success) {
          docCount++;
          liveIds.add(id);
        }
      }
    }
  }
  DirectoryReader reader = w.getReader();
  assertEquals(docCount, reader.numDocs());
  List<LeafReaderContext> leaves = reader.leaves();
  for (LeafReaderContext leafReaderContext : leaves) {
    LeafReader ar = leafReaderContext.reader();
    Bits liveDocs = ar.getLiveDocs();
    int maxDoc = ar.maxDoc();
    for (int i = 0; i < maxDoc; i++) {
      if (liveDocs == null || liveDocs.get(i)) {
        assertTrue(liveIds.remove(ar.document(i).get("id")));
      }
    }
  }
  assertTrue(liveIds.isEmpty());
  w.close();
  IOUtils.close(reader, dir);
}
 

public void testIndexSortWithSparseField() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  SortField sortField = new SortField("dense_int", SortField.Type.INT, true);
  Sort indexSort = new Sort(sortField);
  iwc.setIndexSort(indexSort);
  IndexWriter w = new IndexWriter(dir, iwc);
  Field textField = newTextField("sparse_text", "", Field.Store.NO);
  for (int i = 0; i < 128; i++) {
    Document doc = new Document();
    doc.add(new NumericDocValuesField("dense_int", i));
    if (i < 64) {
      doc.add(new NumericDocValuesField("sparse_int", i));
      doc.add(new BinaryDocValuesField("sparse_binary", new BytesRef(Integer.toString(i))));
      textField.setStringValue("foo");
      doc.add(textField);
    }
    w.addDocument(doc);
  }
  w.commit();
  w.forceMerge(1);
  DirectoryReader r = DirectoryReader.open(w);
  assertEquals(1, r.leaves().size());
  LeafReader leafReader = r.leaves().get(0).reader();

  NumericDocValues denseValues = leafReader.getNumericDocValues("dense_int");
  NumericDocValues sparseValues = leafReader.getNumericDocValues("sparse_int");
  BinaryDocValues sparseBinaryValues = leafReader.getBinaryDocValues("sparse_binary");
  NumericDocValues normsValues = leafReader.getNormValues("sparse_text");
  for(int docID = 0; docID < 128; docID++) {
    assertTrue(denseValues.advanceExact(docID));
    assertEquals(127-docID, (int) denseValues.longValue());
    if (docID >= 64) {
      assertTrue(denseValues.advanceExact(docID));
      assertTrue(sparseValues.advanceExact(docID));
      assertTrue(sparseBinaryValues.advanceExact(docID));
      assertTrue(normsValues.advanceExact(docID));
      assertEquals(1, normsValues.longValue());
      assertEquals(127-docID, (int) sparseValues.longValue());
      assertEquals(new BytesRef(Integer.toString(127-docID)), sparseBinaryValues.binaryValue());
    } else {
      assertFalse(sparseBinaryValues.advanceExact(docID));
      assertFalse(sparseValues.advanceExact(docID));
      assertFalse(normsValues.advanceExact(docID));
    }
  }
  IOUtils.close(r, w, dir);
}
 

@Override
public void tearDown() throws Exception {
  IOUtils.close(indexReader, directory);
  super.tearDown();
}
 

public void testSegmentCountOnFlushRandom() throws Exception {
  Directory dir = newFSDirectory(createTempDir());
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));

  // Never trigger flushes (so we only flush on getReader):
  iwc.setMaxBufferedDocs(100000000);
  iwc.setRAMBufferSizeMB(-1);

  // Never trigger merges (so we can simplistically count flushed segments):
  iwc.setMergePolicy(NoMergePolicy.INSTANCE);

  final IndexWriter w = new IndexWriter(dir, iwc);

  // How many threads are indexing in the current cycle:
  final AtomicInteger indexingCount = new AtomicInteger();

  // How many threads we will use on each cycle:
  final AtomicInteger maxThreadCount = new AtomicInteger();

  CheckSegmentCount checker = new CheckSegmentCount(w, maxThreadCount, indexingCount);

  // We spin up 10 threads up front, but then in between flushes we limit how many can run on each iteration
  final int ITERS = TEST_NIGHTLY ? 300 : 10;
  Thread[] threads = new Thread[MAX_THREADS_AT_ONCE];

  // We use this to stop all threads once they've indexed their docs in the current iter, and pull a new NRT reader, and verify the
  // segment count:
  final CyclicBarrier barrier = new CyclicBarrier(MAX_THREADS_AT_ONCE, checker);
  
  for(int i=0;i<threads.length;i++) {
    threads[i] = new Thread() {
        @Override
        public void run() {
          try {
            for(int iter=0;iter<ITERS;iter++) {
              if (indexingCount.incrementAndGet() <= maxThreadCount.get()) {
                if (VERBOSE) {
                  System.out.println("TEST: " + Thread.currentThread().getName() + ": do index");
                }

                // We get to index on this cycle:
                Document doc = new Document();
                doc.add(new TextField("field", "here is some text that is a bit longer than normal trivial text", Field.Store.NO));
                for(int j=0;j<200;j++) {
                  w.addDocument(doc);
                }
              } else {
                // We lose: no indexing for us on this cycle
                if (VERBOSE) {
                  System.out.println("TEST: " + Thread.currentThread().getName() + ": don't index");
                }
              }
              barrier.await();
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }
        }
      };
    threads[i].start();
  }

  for(Thread t : threads) {
    t.join();
  }

  IOUtils.close(checker, w, dir);
}