org.apache.lucene.index.IndexWriterConfig Java Examples

private IndexReader getReaderWithDocsHavingFamily() throws CorruptIndexException, LockObtainFailedException,
    IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f2", Store.YES));

  Document doc1 = new Document();
  doc1.add(new StringField("a", "b", Store.YES));
  doc1.add(new StringField("family", "f1", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 

@BeforeClass
public static void cleanDatabaseAndInit() throws Exception {

	MongoClient mongo = TestHelper.getMongo();
	mongo.dropDatabase(TestHelper.TEST_DATABASE_NAME);
	directory = new DistributedDirectory(new MongoDirectory(mongo, TestHelper.TEST_DATABASE_NAME, STORAGE_TEST_INDEX, false));

	StandardAnalyzer analyzer = new StandardAnalyzer();
	IndexWriterConfig config = new IndexWriterConfig(analyzer);

	IndexWriter w = new IndexWriter(directory, config);

	addDoc(w, "Random perl Title that is long", "id-1");
	addDoc(w, "Random java Title that is long", "id-1");
	addDoc(w, "MongoDB is awesome", "id-2");
	addDoc(w, "This is a long title with nothing interesting", "id-3");
	addDoc(w, "Java is awesome", "id-4");
	addDoc(w, "Really big fish", "id-5");

	w.commit();
	w.close();
}
 

public static IndexWriter configureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) throws IOException {
  IndexWriterConfig iwc = createWriterConfig(config, runData, mode, commit);
  String infoStreamVal = config.get("writer.info.stream", null);
  if (infoStreamVal != null) {
    if (infoStreamVal.equals("SystemOut")) {
      iwc.setInfoStream(System.out);
    } else if (infoStreamVal.equals("SystemErr")) {
      iwc.setInfoStream(System.err);
    } else {
      Path f = Paths.get(infoStreamVal);
      iwc.setInfoStream(new PrintStream(new BufferedOutputStream(Files.newOutputStream(f)), false, Charset.defaultCharset().name()));
    }
  }
  IndexWriter writer = new IndexWriter(runData.getDirectory(), iwc);
  return writer;
}
 

@Before
public void setupIndex() throws Exception {
    dir = new ByteBuffersDirectory();

    try(IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER))) {
        for (int i = 0; i < docs.length; i++) {
            Document doc = new Document();
            doc.add(new Field("_id", Integer.toString(i + 1), StoredField.TYPE));
            doc.add(newTextField("text", docs[i], Field.Store.YES));
            indexWriter.addDocument(doc);
        }
    }

    reader = DirectoryReader.open(dir);
    searcher = new IndexSearcher(reader);
}
 

/**
 * Add a number of chapters to the index. This is much more efficient than
 * calling add() repeatedly because it just uses one writer rather than
 * opening and closing one for each individual operation.
 *
 * @param bibleList the list of chapters to add.
 */
@Override
public void addAll(Collection<? extends BibleChapter> bibleList) {
    try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(analyzer))) {
        for(BibleChapter chapter : bibleList) {
            Document doc = new Document();
            doc.add(new TextField("text", chapter.getText(), Field.Store.NO));
            doc.add(new TextField("number", Integer.toString(chapter.getID()), Field.Store.YES));
            writer.addDocument(doc);
            chapters.put(chapter.getID(), chapter);
            LOGGER.log(Level.FINE, "Added bible chapter to index: {0}", chapter.getID());
        }
    }
    catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't add value to index", ex);
    }
}
 

@Test
public void testEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());
  
  IOUtils.close(ir, analyzer, dir);
}
 

/**
 * Sets up a RAM-resident Directory, and adds documents (using English.intToEnglish()) with two fields: field and multiField
 * and analyzes them using the PayloadAnalyzer
 * @param similarity The Similarity class to use in the Searcher
 * @param numDocs The num docs to add
 * @return An IndexSearcher
 */
// TODO: randomize
public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) throws IOException {
  Directory directory = new MockDirectoryWrapper(random, new ByteBuffersDirectory());
  PayloadAnalyzer analyzer = new PayloadAnalyzer();

  // TODO randomize this
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
      analyzer).setSimilarity(similarity));
  // writer.infoStream = System.out;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    doc.add(new TextField(FIELD, English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(MULTI_FIELD, English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.forceMerge(1);
  reader = DirectoryReader.open(writer);
  writer.close();

  IndexSearcher searcher = LuceneTestCase.newSearcher(LuceneTestCase.getOnlyLeafReader(reader));
  searcher.setSimilarity(similarity);
  return searcher;
}
 

/** Build the example index. */
public void index() throws IOException {
  IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(
      new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE));

  // Add documents with a fake timestamp, 1000 sec before
  // "now", 2000 sec before "now", ...:
  for(int i=0;i<100;i++) {
    Document doc = new Document();
    long then = nowSec - i * 1000;
    // Add as doc values field, so we can compute range facets:
    doc.add(new NumericDocValuesField("timestamp", then));
    // Add as numeric field so we can drill-down:
    doc.add(new LongPoint("timestamp", then));
    indexWriter.addDocument(doc);
  }

  // Open near-real-time searcher
  searcher = new IndexSearcher(DirectoryReader.open(indexWriter));
  indexWriter.close();
}
 

private static List<String> search(List<String> contents, String query, int n) throws IOException, ParseException {
    List<String> r=new ArrayList<>();
    Directory dir=new RAMDirectory();
    IndexWriter indexWriter=new IndexWriter(dir, new IndexWriterConfig(new EnglishAnalyzer()));
    for (String method:contents){
        Document document=new Document();
        document.add(new TextField("content",method, Field.Store.YES));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    QueryParser qp = new QueryParser("content", new EnglishAnalyzer());
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(dir));
    TopDocs topDocs = indexSearcher.search(qp.parse(query), n);
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
        r.add(indexSearcher.doc(scoreDoc.doc).get("content"));
    }
    return r;
}
 

public void testStressLocks() throws Exception {
  Path tempPath = createTempDir();
  assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));

  Directory dir = getDirectory(tempPath);

  // First create a 1 doc index:
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
  addDoc(w);
  w.close();
  
  int numIterations = atLeast(20);
  WriterThread writer = new WriterThread(numIterations, dir);
  SearcherThread searcher = new SearcherThread(numIterations, dir);
  writer.start();
  searcher.start();

  writer.join();
  searcher.join();

  assertTrue("IndexWriter hit unexpected exceptions", !writer.hitException);
  assertTrue("IndexSearcher hit unexpected exceptions", !searcher.hitException);
  
  dir.close();
}
 

@Override
public void setUp() throws Exception {
  super.setUp();
  dir = new ByteBuffersDirectory();
  appAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(appAnalyzer));
  int numDocs = 200;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    String variedFieldValue = variedFieldValues[i % variedFieldValues.length];
    String repetitiveFieldValue = repetitiveFieldValues[i % repetitiveFieldValues.length];
    doc.add(new TextField("variedField", variedFieldValue, Field.Store.YES));
    doc.add(new TextField("repetitiveField", repetitiveFieldValue, Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.close();
  reader = DirectoryReader.open(dir);
}
 

public void initIndex() throws IOException {
	
	if(dir==null)
		open();
	
	IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
	  				   iwc.setOpenMode(OpenMode.CREATE);
    IndexWriter indexWriter = new IndexWriter(dir, iwc);
    
	for(MagicCard mc : MTGControler.getInstance().getEnabled(MTGCardsProvider.class).listAllCards())
	{
		try {
			indexWriter.addDocument(toDocuments(mc));
		} catch (IllegalArgumentException e) {
			logger.error("Error indexing " + mc + " " + mc.getCurrentSet(),e);
		}
	}
	
	indexWriter.commit();
	indexWriter.close();
}
 

/**
 * Index all text files under a directory.
 */
public void main(final File INDEX_DIR, final String[] args) throws IOException {
  IndexWriterConfig IWConfig = new IndexWriterConfig();
  IWConfig.setOpenMode (IndexWriterConfig.OpenMode.CREATE);
  IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
  IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
  for (int arg = 0; arg < args.length; arg++) {
    final File docDir = new File(args[arg]);
    if (!docDir.exists() || !docDir.canRead()) {
      System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
      throw new IOException("Cannot read from document directory");
    }

    indexDocs(writer, docDir);
    System.out.println("Optimizing...");
    writer.forceMerge(1);
  }
  writer.close();
}
 

@Test
public void testHighCardinalityRatioReturnsTrueForLowCardinality() throws Exception {
    IndexWriter iw = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    String columnName = "x";
    for (int i = 0; i < 10; i++) {
        Document doc = new Document();
        BytesRef value = new BytesRef("1");
        doc.add(new Field(columnName, value, KeywordFieldMapper.Defaults.FIELD_TYPE.clone()));
        iw.addDocument(doc);
    }
    iw.commit();

    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(iw));

    assertThat(
        GroupByOptimizedIterator.hasHighCardinalityRatio(() -> new Engine.Searcher("dummy", indexSearcher, () -> {}), "x"),
        is(false)
    );
}
 

@Test
public void queryFindsADocumentThatWasAdded() throws IOException {

  // Create an in memory lucene index to add a document to
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());

  // Add a document to the lucene index
  Document document = new Document();
  document.add(new TextField("name", "name", Field.Store.YES));
  Field[] fields = SpatialHelper.getIndexableFields(-122.8515139, 45.5099231);
  for (Field field : fields) {
    document.add(field);
  }
  writer.addDocument(document);
  writer.commit();


  // Make sure a findWithin query locates the document
  Query query = SpatialHelper.findWithin(-122.8515239, 45.5099331, 1);
  SearcherManager searcherManager = new SearcherManager(writer, null);
  IndexSearcher searcher = searcherManager.acquire();
  TopDocs results = searcher.search(query, 100);
  assertEquals(1, results.totalHits);
}
 

@Test
public void testMultipleWritersOpenOnSameDirectory() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path,
      "test_multiple"));
  IndexWriter writer1 = new IndexWriter(directory, config.clone());
  addDoc(writer1, getDoc(1));
  IndexWriter writer2 = new IndexWriter(directory, config.clone());
  addDoc(writer2, getDoc(2));
  writer1.close();
  writer2.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  int maxDoc = reader.maxDoc();
  assertEquals(1, maxDoc);
  Document document = reader.document(0);
  assertEquals("2", document.get("id"));
  reader.close();
}
 

public void testRamDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    RAMDirectory ramDirectory = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(ramDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("RamDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(ramDirectory));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RamDirectory search consumes {}ms!", (end - start));
}
 

@Test
public void testMergeUnusedPerFieldCodec() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig iwconf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setOpenMode(OpenMode.CREATE).setCodec(new MockCodec());
  IndexWriter writer = newWriter(dir, iwconf);
  addDocs(writer, 10);
  writer.commit();
  addDocs3(writer, 10);
  writer.commit();
  addDocs2(writer, 10);
  writer.commit();
  assertEquals(30, writer.getDocStats().maxDoc);
  TestUtil.checkIndex(dir);
  writer.forceMerge(1);
  assertEquals(30, writer.getDocStats().maxDoc);
  writer.close();
  dir.close();
}
 

public void testEnsureOpen() throws Exception {
  Directory dir = newDirectory();
  new IndexWriter(dir, new IndexWriterConfig(null)).close();
  SearcherManager sm = new SearcherManager(dir, null);
  IndexSearcher s = sm.acquire();
  sm.close();
  
  // this should succeed;
  sm.release(s);
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.acquire();
  });
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.maybeRefresh();
  });

  dir.close();
}
 

/**
 * Test index creation logic
 */
public void testIndexAndSearchTasks() throws Exception {
  // 1. alg definition (required in every "logic" test)
  String algLines[] = {
      "ResetSystemErase",
      "CreateIndex",
      "{ AddDoc } : 1000",
      "ForceMerge(1)",
      "CloseIndex",
      "OpenReader",
      "{ CountingSearchTest } : 200",
      "CloseReader",
      "[ CountingSearchTest > : 70",
      "[ CountingSearchTest > : 9",
  };
  
  // 2. we test this value later
  CountingSearchTestTask.numSearches = 0;
  
  // 3. execute the algorithm  (required in every "logic" test)
  Benchmark benchmark = execBenchmark(algLines);

  // 4. test specific checks after the benchmark run completed.
  assertEquals("TestSearchTask was supposed to be called!",279,CountingSearchTestTask.numSearches);
  assertTrue("Index does not exist?...!", DirectoryReader.indexExists(benchmark.getRunData().getDirectory()));
  // now we should be able to open the index for write. 
  IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),
      new IndexWriterConfig(new MockAnalyzer(random()))
          .setOpenMode(OpenMode.APPEND));
  iw.close();
  IndexReader ir = DirectoryReader.open(benchmark.getRunData().getDirectory());
  assertEquals("1000 docs were added to the index, this is what we expect to find!",1000,ir.numDocs());
  ir.close();
}
 

private void runIndexer() {
	if (Files.exists(INDEX_DIR)) {
     throw new IllegalArgumentException(
         "Cannot save index to '" + INDEX_DIR + "' directory, please delete it first");
	}

	Date start = new Date();
	try {
		final File docDir1 = new File(LANGUAGE1);
		final File docDir2 = new File(LANGUAGE2);

		IndexWriterConfig writerConfig = new IndexWriterConfig(new StandardAnalyzer());
		IndexWriter writer = new IndexWriter(FSDirectory.open(INDEX_DIR), writerConfig);
		System.out.println("Indexing to directory '" + INDEX_DIR + "'...");
		runDeepIndexer(docDir1, docDir2, writer);

		System.out.println("Optimizing...");
		writer.close();

		Date end = new Date();
		System.out.println(end.getTime() - start.getTime() + " total milliseconds");

	} catch (IOException e) {
		System.out.println(" caught a " + e.getClass() +
				"\n with message: " + e.getMessage());
	}
}
 

/**
 * 创建磁盘目录
 */
public void createFSDirectory(String path) throws Exception {
	this.directory =  FSDirectory.open(new File(path));
	IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_44, this.analyzer);
	IndexWriter indexWriter = new IndexWriter(this.directory, indexWriterConfig);
	indexWriter.close();
}
 

protected static void createIndex(boolean doMultiSegment) throws Exception {
  if (VERBOSE) {
    System.out.println("TEST: setUp");
  }
  // prepare a small index with just a few documents.
  dir = newDirectory();
  anlzr = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(anlzr).setMergePolicy(newLogMergePolicy());
  if (doMultiSegment) {
    iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 2, 7));
  }
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
  // add docs not exactly in natural ID order, to verify we do check the order of docs by scores
  int remaining = N_DOCS;
  boolean done[] = new boolean[N_DOCS];
  int i = 0;
  while (remaining > 0) {
    if (done[i]) {
      throw new Exception("to set this test correctly N_DOCS=" + N_DOCS + " must be primary and greater than 2!");
    }
    addDoc(iw, i);
    done[i] = true;
    i = (i + 4) % N_DOCS;
    remaining --;
  }
  if (!doMultiSegment) {
    if (VERBOSE) {
      System.out.println("TEST: setUp full merge");
    }
    iw.forceMerge(1);
  }
  iw.close();
  if (VERBOSE) {
    System.out.println("TEST: setUp done close");
  }
}
 

public void verifyCompiles() throws Exception {
  Analyzer analyzer = null;

  Directory fsDir = FSDirectory.open(createTempDir("verify"));
  NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 2.0, 25.0);
  IndexWriterConfig conf = new IndexWriterConfig(analyzer);
  IndexWriter writer = new IndexWriter(cachedFSDir, conf);
  writer.close();
  cachedFSDir.close();
}
 

@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newFSDirectory(createTempDir("testDFBlockSize"));
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));
  iw = new RandomIndexWriter(random(), dir, iwc);
  iw.setDoRandomForceMerge(false); // we will ourselves
}
 

public void open() throws Exception
{
    if ( !directory.exists() && !directory.mkdirs() )
    {
        throw new IOException("Could not make: " + directory);
    }

    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE);

    niofsDirectory = new NIOFSDirectory(directory, new SingleInstanceLockFactory());
    writer = new IndexWriter(niofsDirectory, conf);
}
 

/**
 * test that even if a field is not indexed with points, optimized sort still works as expected,
 * although no optimization will be run
 */
public void testLongSortOptimizationOnFieldNotIndexedWithPoints() throws IOException {
  final Directory dir = newDirectory();
  final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());
  final int numDocs = atLeast(100);
  // my_field is not indexed with points
  for (int i = 0; i < numDocs; ++i) {
    final Document doc = new Document();
    doc.add(new NumericDocValuesField("my_field", i));
    writer.addDocument(doc);
  }
  final IndexReader reader = DirectoryReader.open(writer);
  IndexSearcher searcher = new IndexSearcher(reader);
  final SortField sortField = new SortField("my_field", SortField.Type.LONG);
  final Sort sort = new Sort(sortField);
  final int numHits = 3;
  final int totalHitsThreshold = 3;

  final TopFieldCollector collector = TopFieldCollector.create(sort, numHits, null, totalHitsThreshold);
  searcher.search(new MatchAllDocsQuery(), collector);
  TopDocs topDocs = collector.topDocs();
  assertEquals(topDocs.scoreDocs.length, numHits);  // sort still works and returns expected number of docs
  for (int i = 0; i < numHits; i++) {
    FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i];
    assertEquals(i, ((Long) fieldDoc.fields[0]).intValue()); // returns expected values
  }
  assertEquals(topDocs.totalHits.value, numDocs); // assert that all documents were collected => optimization was not run

  writer.close();
  reader.close();
  dir.close();
}
 

public void test() throws IOException {
  MiniCluster miniCluster = new MiniCluster();
  miniCluster.startDfs("./tmp/hdfs");
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43));
  SharedMergeScheduler sharedMergeScheduler = new SharedMergeScheduler(10);
  conf.setMergeScheduler(sharedMergeScheduler.getMergeScheduler());
  Configuration configuration = new Configuration();
  URI fileSystemUri = miniCluster.getFileSystemUri();
  Path path = new Path(fileSystemUri.toString() + "/merge-test");
  FileSystem fileSystem = path.getFileSystem(configuration);
  fileSystem.delete(path, true);
  HdfsDirectory directory = new HdfsDirectory(configuration, path);
  BlurConfiguration blurConfiguration = new BlurConfiguration();
  BlockCacheDirectoryFactoryV2 factory = new BlockCacheDirectoryFactoryV2(blurConfiguration, 1000000l);

  Directory cdir = factory.newDirectory("t", "s", directory, null);
  IndexWriter writer = new IndexWriter(cdir, conf);
  Random random = new Random(1);
  StringBuilder stringBuilder = new StringBuilder();
  long s = System.nanoTime();
  for (int i = 0; i < 250000; i++) {
    if (i % 5000 == 0) {
      System.out.println(i);
    }
    stringBuilder.setLength(0);
    for (int w = 0; w < 2000; w++) {
      stringBuilder.append(Integer.toString(random.nextInt(100000))).append(' ');
    }
    Document document = new Document();
    document.add(new TextField("body", stringBuilder.toString(), Store.YES));
    writer.addDocument(document);
  }
  writer.close(true);
  sharedMergeScheduler.close();
  factory.close();
  long e = System.nanoTime();
  System.out.println("Total Time [" + (e - s) / 1000000.0 + " ms]");
  miniCluster.shutdownDfs();
}
 

@Test
public void testMultiThreadedLuceneRealtime() throws Exception {
  File indexFile = new File(INDEX_DIR.getPath() + "/realtime-test3.index");
  Directory indexDirectory = FSDirectory.open(indexFile.toPath());
  StandardAnalyzer standardAnalyzer = new StandardAnalyzer();
  // create and open a writer
  IndexWriterConfig indexWriterConfig = new IndexWriterConfig(standardAnalyzer);
  indexWriterConfig.setRAMBufferSizeMB(500);
  IndexWriter indexWriter = new IndexWriter(indexDirectory, indexWriterConfig);

  // create an NRT index reader
  SearcherManager searcherManager = new SearcherManager(indexWriter, false, false, null);

  // background thread to refresh NRT reader
  ControlledRealTimeReopenThread controlledRealTimeReopenThread =
      new ControlledRealTimeReopenThread(indexWriter, searcherManager, 0.01, 0.01);
  controlledRealTimeReopenThread.start();

  // start writer and reader
  Thread writer = new Thread(new RealtimeWriter(indexWriter));
  Thread realtimeReader = new Thread(new RealtimeReader(searcherManager, standardAnalyzer));

  writer.start();
  realtimeReader.start();

  writer.join();
  realtimeReader.join();
  controlledRealTimeReopenThread.join();
}
 

private IndexWriter newWriter(Directory dir, IndexWriterConfig conf)
    throws IOException {
  LogDocMergePolicy logByteSizeMergePolicy = new LogDocMergePolicy();
  logByteSizeMergePolicy.setNoCFSRatio(0.0); // make sure we use plain
  // files
  conf.setMergePolicy(logByteSizeMergePolicy);

  final IndexWriter writer = new IndexWriter(dir, conf);
  return writer;
}