Java Code Examples for org.apache.lucene.util.Version#LUCENE_CURRENT

protected static boolean clearIndex(File path) {
	try {
		if (path == null || !path.exists())
			return true; // if path doesnt exist, then there is nothing to
							// clear

		FSDirectory indexDir = new FSDirectoryProvider().getDirectory();
		IndexWriter writer = new IndexWriter(indexDir.open(path),
				new IndexWriterConfig(Version.LUCENE_CURRENT,
						new WhitespaceAnalyzer(Version.LUCENE_CURRENT)));
		writer.deleteAll();
		writer.close();
		return true;
	} catch (Exception ex) {
		org.webdsl.logging.Logger.error(
				"Error while clearing index on location: " + path, ex);
		return false;
	}

}
 

/**
 * Use a different index as the auto completer index or re-open
 * the existing index if <code>autocompleteIndex</code> is the same value
 * as given in the constructor.
 * @param autocompleteIndexDir the autocomplete directory to use
 * @throws AlreadyClosedException if the Autocompleter is already closed
 * @throws  IOException if autocompleter can not open the directory
 */
// TODO: we should make this final as it is called in the constructor
public void setAutoCompleteIndex(Directory autocompleteIndexDir) throws IOException {
  // this could be the same directory as the current autocompleteIndex
  // modifications to the directory should be synchronized
  synchronized (modifyCurrentIndexLock) {
    ensureOpen();
    if (!IndexReader.indexExists(autocompleteIndexDir)) {
        IndexWriter writer = new IndexWriter(autocompleteIndexDir,
          new IndexWriterConfig(Version.LUCENE_CURRENT,
              new WhitespaceAnalyzer(Version.LUCENE_CURRENT)));
        writer.close();
    }
    swapSearcher(autocompleteIndexDir);
  }
}
 

protected Set<String> getHighlightWords(final String searchString) {
    try {
        final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
        final TokenStream stream = analyzer.tokenStream("content", new StringReader(searchString));
        final TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
        for (boolean next = stream.incrementToken(); next; next = stream.incrementToken()) {
            final String term = termAtt.term();
            if (log.isDebugEnabled()) {
                log.debug(term);
            }
        }
    } catch (final IOException e) {
        log.error("", e);
    }
    return null;
}
 

/**
 * [used by spring]
 */
private SearchServiceImpl(final SearchModule searchModule, final MainIndexer mainIndexer, final JmsSearchProvider searchProvider) {
    log.info("Start SearchServiceImpl constructor...");
    this.searchModuleConfig = searchModule;
    this.mainIndexer = mainIndexer;
    analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
    searchProvider.setSearchService(this);

}
 

/**
 * @param id
 *            Unique index ID. Is used to generate unique directory name.
 * @param tempIndexPath
 *            Absolute directory-path where the temporary index can be generated.
 * @param fullIndexer
 *            Reference to full-index
 */
public IndexWriterWorker(final int id, final File tempIndexDir, final OlatFullIndexer fullIndexer) {
    this.id = id;
    this.fullIndexer = fullIndexer;
    try {
        final File indexPartFile = new File(tempIndexDir, "part" + id);
        final Directory indexPartDirectory = FSDirectory.open(indexPartFile);
        indexWriter = new IndexWriter(indexPartDirectory, new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
        indexWriter.deleteAll();
    } catch (final IOException e) {
        log.warn("Can not create IndexWriter");
    }
}
 

/**
 *
 * @param text
 * @return
 * @throws IOException
 */
public Map<String, Float> buildBag(String text) throws IOException {
    Map<String, Float> bag = new HashMap<>();
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
    SnowballStemmer stemmer = null;
    if (stemming) {
        stemmer = getStemmer(language);
        if (stemmer == null) {
            Logger.getLogger(RevisedLesk.class.getName()).log(Level.WARNING, "No stemmer for language {0}", language);
        }
    }
    TokenStream tokenStream = analyzer.tokenStream("gloss", new StringReader(text));
    while (tokenStream.incrementToken()) {
        TermAttribute token = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
        String term = token.term();
        if (stemmer != null) {
            stemmer.setCurrent(term);
            if (stemmer.stem()) {
                term = stemmer.getCurrent();
            }
        }
        Float c = bag.get(term);
        if (c == null) {
            bag.put(term, 1f);
        } else {
            bag.put(term, c + 1f);
        }
    }
    return bag;
}
 

public VersionSetAnalyzerPair(ConfigOptionsToAnalyzer lro,
		Class<? extends Analyzer> cls) throws Exception {
	super(lro.languageRange, getConstructor(cls, Version.class, Set.class), Version.LUCENE_CURRENT, lro.getStopWords());
}
 

/**
 * Do search a certain query. The results will be filtered for the identity and roles.
 * 
 * @param queryString
 *            Search query-string.
 * @param identity
 *            Filter results for this identity (user).
 * @param roles
 *            Filter results for this roles (role of user).
 * @return SearchResults object for this query
 */
@Override
public SearchResults doSearch(final String queryString, final List<String> condQueries, final Identity identity, final Roles roles, final int firstResult,
        final int maxResults, final boolean doHighlighting) throws ServiceNotAvailableException, ParseException {
    try {
        if (!existIndex()) {
            log.warn("Index does not exist, can't search for queryString: " + queryString);
            throw new ServiceNotAvailableException("Index does not exist");
        }
        synchronized (createIndexSearcherLock) {// o_clusterOK by:fj if service is only configured on one vm, which is recommended way
            if (searcher == null) {
                try {
                    createIndexSearcher(indexPath);
                    checkIsIndexUpToDate();
                } catch (final IOException ioEx) {
                    log.warn("Can not create searcher", ioEx);
                    throw new ServiceNotAvailableException("Index is not available");
                }
            }
            if (hasNewerIndexFile()) {
                reopenIndexSearcher();
                checkIsIndexUpToDate();
            }
        }
        log.info("queryString=" + queryString);

        final BooleanQuery query = new BooleanQuery();
        if (StringHelper.containsNonWhitespace(queryString)) {
            final QueryParser queryParser = new MultiFieldQueryParser(Version.LUCENE_CURRENT, fields, analyzer);
            queryParser.setLowercaseExpandedTerms(false);// some add. fields are not tokenized and not lowered case
            final Query multiFieldQuery = queryParser.parse(queryString.toLowerCase());
            query.add(multiFieldQuery, Occur.MUST);
        }

        if (condQueries != null && !condQueries.isEmpty()) {
            for (final String condQueryString : condQueries) {
                final QueryParser condQueryParser = new QueryParser(Version.LUCENE_CURRENT, condQueryString, analyzer);
                condQueryParser.setLowercaseExpandedTerms(false);
                final Query condQuery = condQueryParser.parse(condQueryString);
                query.add(condQuery, Occur.MUST);
            }
        }

        if (log.isDebugEnabled()) {
            log.debug("query=" + query);
        }
        // TODO: 14.06.2010/cg : fellowig cide fragment can be removed later, do no longer call rewrite(query) because wildcard-search problem (OLAT-5359)
        // Query query = null;
        // try {
        // query = searcher.rewrite(query);
        // log.debug("after 'searcher.rewrite(query)' query=" + query);
        // } catch (Exception ex) {
        // throw new QueryException("Rewrite-Exception query because too many clauses. Query=" + query);
        // }
        final long startTime = System.currentTimeMillis();
        final int n = SearchServiceFactory.getService().getSearchModuleConfig().getMaxHits();
        final TopDocs docs = searcher.search(query, n);
        final long queryTime = System.currentTimeMillis() - startTime;
        if (log.isDebugEnabled()) {
            log.debug("hits.length()=" + docs.totalHits);
        }
        final SearchResultsImpl searchResult = new SearchResultsImpl(mainIndexer, searcher, docs, query, analyzer, identity, roles, firstResult, maxResults,
                doHighlighting);
        searchResult.setQueryTime(queryTime);
        searchResult.setNumberOfIndexDocuments(searcher.maxDoc());
        queryCount++;
        return searchResult;
    } catch (final ServiceNotAvailableException naex) {
        // pass exception
        throw new ServiceNotAvailableException(naex.getMessage());
    } catch (final ParseException pex) {
        throw new ParseException("can not parse query=" + queryString);
    } catch (final Exception ex) {
        log.warn("Exception in search", ex);
        throw new ServiceNotAvailableException(ex.getMessage());
    }
}
 

/**
 * Creates a new spell-check index based on search-index
 */
public void createSpellIndex() {
    if (isSpellCheckEnabled) {
        IndexReader indexReader = null;
        try {
            log.info("Start generating Spell-Index...");
            long startSpellIndexTime = 0;
            if (log.isDebugEnabled()) {
                startSpellIndexTime = System.currentTimeMillis();
            }
            final Directory indexDir = FSDirectory.open(new File(indexPath));
            indexReader = IndexReader.open(indexDir);
            // 1. Create content spellIndex
            final File spellDictionaryFile = new File(spellDictionaryPath);
            final Directory contentSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + CONTENT_PATH));// true
            final SpellChecker contentSpellChecker = new SpellChecker(contentSpellIndexDirectory);
            final Dictionary contentDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.CONTENT_FIELD_NAME);
            contentSpellChecker.indexDictionary(contentDictionary);
            // 2. Create title spellIndex
            final Directory titleSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + TITLE_PATH));// true
            final SpellChecker titleSpellChecker = new SpellChecker(titleSpellIndexDirectory);
            final Dictionary titleDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.TITLE_FIELD_NAME);
            titleSpellChecker.indexDictionary(titleDictionary);
            // 3. Create description spellIndex
            final Directory descriptionSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + DESCRIPTION_PATH));// true
            final SpellChecker descriptionSpellChecker = new SpellChecker(descriptionSpellIndexDirectory);
            final Dictionary descriptionDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.DESCRIPTION_FIELD_NAME);
            descriptionSpellChecker.indexDictionary(descriptionDictionary);
            // 4. Create author spellIndex
            final Directory authorSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + AUTHOR_PATH));// true
            final SpellChecker authorSpellChecker = new SpellChecker(authorSpellIndexDirectory);
            final Dictionary authorDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.AUTHOR_FIELD_NAME);
            authorSpellChecker.indexDictionary(authorDictionary);

            // Merge all part spell indexes (content,title etc.) to one common spell index
            final Directory spellIndexDirectory = FSDirectory.open(spellDictionaryFile);// true
            final IndexWriter merger = new IndexWriter(spellIndexDirectory, new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
            final Directory[] directories = { contentSpellIndexDirectory, titleSpellIndexDirectory, descriptionSpellIndexDirectory, authorSpellIndexDirectory };
            merger.addIndexesNoOptimize(directories);
            merger.optimize();
            merger.close();
            spellChecker = new SpellChecker(spellIndexDirectory);
            spellChecker.setAccuracy(0.7f);
            if (log.isDebugEnabled()) {
                log.debug("SpellIndex created in " + (System.currentTimeMillis() - startSpellIndexTime) + "ms");
            }
            log.info("New generated Spell-Index ready to use.");
        } catch (final IOException ioEx) {
            log.warn("Can not create SpellIndex", ioEx);
        } finally {
            if (indexReader != null) {
                try {
                    indexReader.close();
                } catch (final IOException e) {
                    log.warn("Can not close indexReader properly", e);
                }
            }
        }
    }
}
 

/**
 * Create index-writer object. In multi-threaded mode ctreates an array of index-workers. Start indexing with main-index as root object. Index recursive all elements.
 * At the end optimze and close new index. The new index is stored in [temporary-index-path]/main
 * 
 * @throws InterruptedException
 */
private void doIndex() throws InterruptedException {
    try {
        final File tempIndexDir = new File(tempIndexPath);
        final Directory indexPath = FSDirectory.open(new File(tempIndexDir, "main"));
        final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
        indexWriter = new IndexWriter(indexPath, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
        indexWriter.deleteAll();
        indexWriter.setMergeFactor(INDEX_MERGE_FACTOR); // for better performance
        indexWriter.setRAMBufferSizeMB(ramBufferSizeMB);// for better performance set to 48MB (see lucene docu 'how to make indexing faster")
        log.info("IndexWriter config RAMBufferSizeMB=" + indexWriter.getRAMBufferSizeMB());
        indexWriter.setUseCompoundFile(useCompoundFile); // for better performance (see lucene docu 'how to make indexing faster")
        log.info("IndexWriter config UseCompoundFile=" + indexWriter.getUseCompoundFile());
        // Create IndexWriterWorker
        log.info("Running with " + numberIndexWriter + " IndexerWriterWorker");
        indexWriterWorkers = new IndexWriterWorker[numberIndexWriter];
        final Directory[] partIndexDirs = new Directory[numberIndexWriter];
        for (int i = 0; i < numberIndexWriter; i++) {
            final IndexWriterWorker indexWriterWorker = new IndexWriterWorker(i, tempIndexDir, this);
            indexWriterWorkers[i] = indexWriterWorker;
            indexWriterWorkers[i].start();
            partIndexDirs[i] = indexWriterWorkers[i].getIndexDir();
        }

        final SearchResourceContext searchResourceContext = new SearchResourceContext();
        log.info("doIndex start. OlatFullIndexer with Debug output");
        mainIndexer.doIndex(searchResourceContext, null /* no parent */, this);

        log.info("Wait until every folder indexer is finished");

        DBFactory.getInstance().commitAndCloseSession();
        // check if every folder indexer is finished max waiting-time 10Min (=waitingCount-limit = 60)
        int waitingCount = 0;
        final int MAX_WAITING_COUNT = 60;// = 10Min
        while (FolderIndexerWorkerPool.getInstance().isIndexerRunning() && (waitingCount++ < MAX_WAITING_COUNT)) {
            Thread.sleep(10000);
        }
        if (waitingCount >= MAX_WAITING_COUNT) {
            log.info("Finished with max waiting time!");
        }
        log.info("Set Finish-flag for each indexWriterWorkers");
        // Set Finish-flag
        for (int i = 0; i < numberIndexWriter; i++) {
            indexWriterWorkers[i].finishIndexing();
        }

        log.info("Wait until every indexworker is finished");
        // check if every indexworker is finished max waiting-time 10Min (=waitingCount-limit = 60)
        waitingCount = 0;
        while (!areIndexingDone() && (waitingCount++ < MAX_WAITING_COUNT)) {
            Thread.sleep(10000);
        }
        if (waitingCount >= MAX_WAITING_COUNT) {
            log.info("Finished with max waiting time!");
        }

        // Merge all partIndex
        DBFactory.getInstance().commitAndCloseSession();
        if (partIndexDirs.length > 0) {
            log.info("Start merging part Indexes");
            indexWriter.addIndexesNoOptimize(partIndexDirs);
            log.info("Added all part Indexes");
        }
        fullIndexerStatus.setIndexSize(indexWriter.maxDoc());
        indexWriter.optimize();
        indexWriter.close();
    } catch (final IOException e) {
        e.printStackTrace();
        log.warn("Can not create IndexWriter, indexname=" + tempIndexPath, e);
    } finally {
        DBFactory.getInstance().commitAndCloseSession();
        log.debug("doIndex: commit & close session");
    }
}