Java Code Examples for org.apache.lucene.index.IndexWriter#setUseCompoundFile()

/**
 * @param id
 *            Unique index ID. Is used to generate unique directory name.
 * @param tempIndexPath
 *            Absolute directory-path where the temporary index can be generated.
 * @param fullIndexer
 *            Reference to full-index
 */
public IndexWriterWorker(final int id, final File tempIndexDir, final OlatFullIndexer fullIndexer) {
    this.id = id;
    this.indexPartDir = new File(tempIndexDir, "part" + id);
    this.fullIndexer = fullIndexer;
    try {
        final Directory luceneIndexPartDir = FSDirectory.open(indexPartDir);
        indexWriter = new IndexWriter(luceneIndexPartDir, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
        indexWriter.setMergeFactor(fullIndexer.getSearchModuleConfig().getIndexerWriterMergeFactor());
        log.info("IndexWriter config MergeFactor=" + indexWriter.getMergeFactor());
        indexWriter.setRAMBufferSizeMB(fullIndexer.getSearchModuleConfig().getIndexerWriterRamBuffer());
        log.info("IndexWriter config RAMBufferSizeMB=" + indexWriter.getRAMBufferSizeMB());
        indexWriter.setUseCompoundFile(false);
    } catch (final IOException e) {
        log.warn("Can not create IndexWriter");
    }
}
 

private IndexWriter getIndexWriter(String name, String lang)
        throws CorruptIndexException, LockObtainFailedException,
        IOException {
    String path = indexWorkDir + name;
    File f = new File(path);
    f.mkdirs();
    Analyzer analyzer = getAnalyzer(name, lang);
    IndexWriter writer = new IndexWriter(path, analyzer);
    writer.setUseCompoundFile(true);
    return writer;
}
 

/**
 * Test of indexing with Lucene.
 * @throws Exception if an error is thrown while executing.
 */
public static void luceneIndex() throws Exception {
  // setting default parameters
  final int depth = 3;

  // create Lucene index writer
  final IndexWriter writer = new IndexWriter(index, new StandardAnalyzer(), true);
  writer.setUseCompoundFile(true);
  writer.setMaxFieldLength(1000000);

  // common crawler settings
  final Crawler crawler = new Crawler();
  crawler.setLinkFilter(new ServerFilter(server));
  crawler.setModel(new MaxDepthModel(depth));
  crawler.addParserListener(new IParserEventListener() {
    @Override
    public void parse(final ParserEvent event) {
      print("Parsing link: " + event.getLink());
    }
  });

  // create Lucene parsing listener and add it
  final LuceneParserEventListener listener = new LuceneParserEventListener(writer);
  crawler.addParserListener(listener);

  // start crawler
  crawler.start(server, startPage);

  // Optimizing Lucene index
  writer.optimize();
  writer.close();
}
 

private IndexWriter getIndexWriter(String name, String lang)
        throws CorruptIndexException, LockObtainFailedException,
        IOException {
    String path = indexWorkDir + name;
    File f = new File(path);
    f.mkdirs();
    Analyzer analyzer = getAnalyzer(name, lang);
    IndexWriter writer = new IndexWriter(path, analyzer);
    writer.setUseCompoundFile(true);
    return writer;
}
 

private IndexWriter createWriter() throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, null,
          new KeepOnlyLastCommitDeletionPolicy());
  writer.setUseCompoundFile(false);

  if (iconf != null) {
    int maxFieldLength = iconf.getIndexMaxFieldLength();
    if (maxFieldLength > 0) {
      writer.setMaxFieldLength(maxFieldLength);
    }
  }

  return writer;
}
 

private IndexWriter createWriter() throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, null,
          new KeepOnlyLastCommitDeletionPolicy());
  writer.setUseCompoundFile(false);

  if (iconf != null) {
    int maxFieldLength = iconf.getIndexMaxFieldLength();
    if (maxFieldLength > 0) {
      writer.setMaxFieldLength(maxFieldLength);
    }
  }

  return writer;
}
 

/**
 * Create index-writer object. In multi-threaded mode ctreates an array of index-workers. Start indexing with main-index as root object. Index recursive all elements.
 * At the end optimze and close new index. The new index is stored in [temporary-index-path]/main
 * 
 * @throws InterruptedException
 */
private void doIndex() throws InterruptedException {
    try {
        final File tempIndexDir = new File(tempIndexPath);
        final Directory indexPath = FSDirectory.open(new File(tempIndexDir, "main"));
        final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
        indexWriter = new IndexWriter(indexPath, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
        indexWriter.deleteAll();
        indexWriter.setMergeFactor(INDEX_MERGE_FACTOR); // for better performance
        indexWriter.setRAMBufferSizeMB(ramBufferSizeMB);// for better performance set to 48MB (see lucene docu 'how to make indexing faster")
        log.info("IndexWriter config RAMBufferSizeMB=" + indexWriter.getRAMBufferSizeMB());
        indexWriter.setUseCompoundFile(useCompoundFile); // for better performance (see lucene docu 'how to make indexing faster")
        log.info("IndexWriter config UseCompoundFile=" + indexWriter.getUseCompoundFile());
        // Create IndexWriterWorker
        log.info("Running with " + numberIndexWriter + " IndexerWriterWorker");
        indexWriterWorkers = new IndexWriterWorker[numberIndexWriter];
        final Directory[] partIndexDirs = new Directory[numberIndexWriter];
        for (int i = 0; i < numberIndexWriter; i++) {
            final IndexWriterWorker indexWriterWorker = new IndexWriterWorker(i, tempIndexDir, this);
            indexWriterWorkers[i] = indexWriterWorker;
            indexWriterWorkers[i].start();
            partIndexDirs[i] = indexWriterWorkers[i].getIndexDir();
        }

        final SearchResourceContext searchResourceContext = new SearchResourceContext();
        log.info("doIndex start. OlatFullIndexer with Debug output");
        mainIndexer.doIndex(searchResourceContext, null /* no parent */, this);

        log.info("Wait until every folder indexer is finished");

        DBFactory.getInstance().commitAndCloseSession();
        // check if every folder indexer is finished max waiting-time 10Min (=waitingCount-limit = 60)
        int waitingCount = 0;
        final int MAX_WAITING_COUNT = 60;// = 10Min
        while (FolderIndexerWorkerPool.getInstance().isIndexerRunning() && (waitingCount++ < MAX_WAITING_COUNT)) {
            Thread.sleep(10000);
        }
        if (waitingCount >= MAX_WAITING_COUNT) {
            log.info("Finished with max waiting time!");
        }
        log.info("Set Finish-flag for each indexWriterWorkers");
        // Set Finish-flag
        for (int i = 0; i < numberIndexWriter; i++) {
            indexWriterWorkers[i].finishIndexing();
        }

        log.info("Wait until every indexworker is finished");
        // check if every indexworker is finished max waiting-time 10Min (=waitingCount-limit = 60)
        waitingCount = 0;
        while (!areIndexingDone() && (waitingCount++ < MAX_WAITING_COUNT)) {
            Thread.sleep(10000);
        }
        if (waitingCount >= MAX_WAITING_COUNT) {
            log.info("Finished with max waiting time!");
        }

        // Merge all partIndex
        DBFactory.getInstance().commitAndCloseSession();
        if (partIndexDirs.length > 0) {
            log.info("Start merging part Indexes");
            indexWriter.addIndexesNoOptimize(partIndexDirs);
            log.info("Added all part Indexes");
        }
        fullIndexerStatus.setIndexSize(indexWriter.maxDoc());
        indexWriter.optimize();
        indexWriter.close();
    } catch (final IOException e) {
        e.printStackTrace();
        log.warn("Can not create IndexWriter, indexname=" + tempIndexPath, e);
    } finally {
        DBFactory.getInstance().commitAndCloseSession();
        log.debug("doIndex: commit & close session");
    }
}