Java Code Examples for org.apache.lucene.util.Version#LUCENE_30

/**
 * @param id
 *            Unique index ID. Is used to generate unique directory name.
 * @param tempIndexPath
 *            Absolute directory-path where the temporary index can be generated.
 * @param fullIndexer
 *            Reference to full-index
 */
public IndexWriterWorker(final int id, final File tempIndexDir, final OlatFullIndexer fullIndexer) {
    this.id = id;
    this.indexPartDir = new File(tempIndexDir, "part" + id);
    this.fullIndexer = fullIndexer;
    try {
        final Directory luceneIndexPartDir = FSDirectory.open(indexPartDir);
        indexWriter = new IndexWriter(luceneIndexPartDir, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
        indexWriter.setMergeFactor(fullIndexer.getSearchModuleConfig().getIndexerWriterMergeFactor());
        log.info("IndexWriter config MergeFactor=" + indexWriter.getMergeFactor());
        indexWriter.setRAMBufferSizeMB(fullIndexer.getSearchModuleConfig().getIndexerWriterRamBuffer());
        log.info("IndexWriter config RAMBufferSizeMB=" + indexWriter.getRAMBufferSizeMB());
        indexWriter.setUseCompoundFile(false);
    } catch (final IOException e) {
        log.warn("Can not create IndexWriter");
    }
}
 

@Transactional(readOnly = true)
public Integer createIndex(Integer siteId, Integer channelId,
		Date startDate, Date endDate, Integer startId, Integer max,
		Directory dir) throws IOException, ParseException {
	boolean exist = IndexReader.indexExists(dir);
	IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
			Version.LUCENE_30), !exist, IndexWriter.MaxFieldLength.LIMITED);
	try {
		if (exist) {
			LuceneContent.delete(siteId, channelId, startDate, endDate,
					writer);
		}
		Integer lastId = luceneContentDao.index(writer, siteId, channelId,
				startDate, endDate, startId, max);
		writer.optimize();
		return lastId;
	} finally {
		writer.close();
	}
}
 

@Transactional(readOnly = true)
public Pagination searchPage(Directory dir, String queryString,String category,String workplace,
		Integer siteId, Integer channelId, Date startDate, Date endDate,
		int pageNo, int pageSize) throws CorruptIndexException,
		IOException, ParseException {
	Searcher searcher = new IndexSearcher(dir);
	try {
		Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
		Query query = LuceneContent.createQuery(queryString,category,workplace, siteId,
				channelId, startDate, endDate, analyzer);
		TopDocs docs = searcher.search(query, pageNo * pageSize);
		Pagination p = LuceneContent.getResultPage(searcher, docs, pageNo,
				pageSize);
		List<?> ids = p.getList();
		List<Content> contents = new ArrayList<Content>(ids.size());
		for (Object id : ids) {
			contents.add(contentMng.findById((Integer) id));
		}
		p.setList(contents);
		return p;
	} finally {
		searcher.close();
	}
}
 

public Index(final SearchModule searchModule, final MainIndexer mainIndexer, final long maxIndexTime, final String[] fields) {
    this.searchModule = searchModule;
    this.mainIndexer = mainIndexer;
    this.maxIndexTime = maxIndexTime;
    this.fields = fields;

    fullIndexer = new OlatFullIndexer(this, mainIndexer, searchModule);
    analyzer = new StandardAnalyzer(Version.LUCENE_30);

    createIndexSearcher(false);
    if (searchModule.isSpellCheckEnabled()) {
        createSpellCheckSearcher(false);
    }
}
 

@Transactional(readOnly = true)
public void createIndex(Content content, Directory dir) throws IOException {
	boolean exist = IndexReader.indexExists(dir);
	IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
			Version.LUCENE_30), !exist, IndexWriter.MaxFieldLength.LIMITED);
	try {
		writer.addDocument(LuceneContent.createDocument(content));
	} finally {
		writer.close();
	}
}
 

@Transactional(readOnly = true)
public void deleteIndex(Integer contentId, Directory dir)
		throws IOException, ParseException {
	boolean exist = IndexReader.indexExists(dir);
	if (exist) {
		IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
				Version.LUCENE_30), false,
				IndexWriter.MaxFieldLength.LIMITED);
		try {
			LuceneContent.delete(contentId, writer);
		} finally {
			writer.close();
		}
	}
}
 

public void updateIndex(Content content, Directory dir) throws IOException,
		ParseException {
	boolean exist = IndexReader.indexExists(dir);
	IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
			Version.LUCENE_30), !exist, IndexWriter.MaxFieldLength.LIMITED);
	try {
		if (exist) {
			LuceneContent.delete(content.getId(), writer);
		}
		writer.addDocument(LuceneContent.createDocument(content));
	} finally {
		writer.close();
	}
}
 

@Transactional(readOnly = true)
public List<Content> searchList(Directory dir, String queryString,String category,String workplace,
		Integer siteId, Integer channelId, Date startDate, Date endDate,
		int first, int max) throws CorruptIndexException, IOException,
		ParseException {
	Searcher searcher = new IndexSearcher(dir);
	try {
		Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
		Query query = LuceneContent.createQuery(queryString,category,workplace, siteId,
				channelId, startDate, endDate, analyzer);
		if (first < 0) {
			first = 0;
		}
		if (max < 0) {
			max = 0;
		}
		TopDocs docs = searcher.search(query, first + max);
		List<Integer> ids = LuceneContent.getResultList(searcher, docs,
				first, max);
		List<Content> contents = new ArrayList<Content>(ids.size());
		for (Object id : ids) {
			contents.add(contentMng.findById((Integer) id));
		}
		return contents;
	} finally {
		searcher.close();
	}
}
 

/**
 * Creates a new spell-check index based on search-index
 */
public static void createSpellIndex(final SearchModule searchModule) {
    final String tempSearchIndexPath = searchModule.getTempSearchIndexPath();
    final String tempSpellCheckIndexPath = searchModule.getTempSpellCheckerIndexPath();

    IndexReader indexReader = null;
    try {
        log.info("Start generating spell check index ...");

        long startSpellIndexTime = 0;
        if (log.isDebugEnabled()) {
            startSpellIndexTime = System.currentTimeMillis();
        }
        final Directory indexDir = FSDirectory.open(new File(tempSearchIndexPath, "main"));
        indexReader = IndexReader.open(indexDir);

        // 1. Create content spellIndex
        log.info("Generating 'content' spell check index ...");
        final File contentSpellIndexPath = new File(tempSpellCheckIndexPath + CONTENT_PATH);
        FileUtils.deleteDirsAndFiles(contentSpellIndexPath, true, true);
        final Directory contentSpellIndexDirectory = FSDirectory.open(contentSpellIndexPath);
        final SpellChecker contentSpellChecker = new SpellChecker(contentSpellIndexDirectory);
        final Dictionary contentDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.CONTENT_FIELD_NAME);
        contentSpellChecker.indexDictionary(contentDictionary);

        // 2. Create title spellIndex
        log.info("Generating 'title' spell check index ...");
        final File titleSpellIndexPath = new File(tempSpellCheckIndexPath + TITLE_PATH);
        FileUtils.deleteDirsAndFiles(titleSpellIndexPath, true, true);
        final Directory titleSpellIndexDirectory = FSDirectory.open(titleSpellIndexPath);
        final SpellChecker titleSpellChecker = new SpellChecker(titleSpellIndexDirectory);
        final Dictionary titleDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.TITLE_FIELD_NAME);
        titleSpellChecker.indexDictionary(titleDictionary);

        // 3. Create description spellIndex
        log.info("Generating 'description' spell check index ...");
        final File descriptionSpellIndexPath = new File(tempSpellCheckIndexPath + DESCRIPTION_PATH);
        FileUtils.deleteDirsAndFiles(descriptionSpellIndexPath, true, true);
        final Directory descriptionSpellIndexDirectory = FSDirectory.open(descriptionSpellIndexPath);
        final SpellChecker descriptionSpellChecker = new SpellChecker(descriptionSpellIndexDirectory);
        final Dictionary descriptionDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.DESCRIPTION_FIELD_NAME);
        descriptionSpellChecker.indexDictionary(descriptionDictionary);

        // 4. Create author spellIndex
        log.info("Generating 'author' spell check index ...");
        final File authorSpellIndexPath = new File(tempSpellCheckIndexPath + AUTHOR_PATH);
        FileUtils.deleteDirsAndFiles(authorSpellIndexPath, true, true);
        final Directory authorSpellIndexDirectory = FSDirectory.open(authorSpellIndexPath);
        final SpellChecker authorSpellChecker = new SpellChecker(authorSpellIndexDirectory);
        final Dictionary authorDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.AUTHOR_FIELD_NAME);
        authorSpellChecker.indexDictionary(authorDictionary);

        log.info("Merging spell check indices ...");
        // Merge all part spell indexes (content,title etc.) to one common spell index
        final File tempSpellCheckIndexDir = new File(tempSpellCheckIndexPath);
        FileUtils.deleteDirsAndFiles(tempSpellCheckIndexDir, true, true);
        final Directory tempSpellIndexDirectory = FSDirectory.open(tempSpellCheckIndexDir);
        final IndexWriter merger = new IndexWriter(tempSpellIndexDirectory, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
        final Directory[] directories = { contentSpellIndexDirectory, titleSpellIndexDirectory, descriptionSpellIndexDirectory, authorSpellIndexDirectory };
        merger.addIndexesNoOptimize(directories);

        log.info("Optimizing spell check index ...");
        merger.optimize();
        merger.close();

        tempSpellIndexDirectory.close();

        contentSpellChecker.close();
        contentSpellIndexDirectory.close();

        titleSpellChecker.close();
        titleSpellIndexDirectory.close();

        descriptionSpellChecker.close();
        descriptionSpellIndexDirectory.close();

        authorSpellChecker.close();
        authorSpellIndexDirectory.close();

        FileUtils.deleteDirsAndFiles(contentSpellIndexPath, true, true);
        FileUtils.deleteDirsAndFiles(titleSpellIndexPath, true, true);
        FileUtils.deleteDirsAndFiles(descriptionSpellIndexPath, true, true);
        FileUtils.deleteDirsAndFiles(authorSpellIndexPath, true, true);

        if (log.isDebugEnabled()) {
            log.debug("Spell check index created in " + (System.currentTimeMillis() - startSpellIndexTime) + " ms.");
        }
    } catch (final IOException ioEx) {
        log.warn("Can not create spell check index.", ioEx);
    } finally {
        if (indexReader != null) {
            try {
                indexReader.close();
            } catch (final IOException e) {
                log.warn("Can not close indexReader properly", e);
            }
        }
    }
}
 

/**
 * @see org.olat.lms.search.SearchService#doSearch(String, List, Identity, Roles, int, int, boolean)
 */
public SearchResults doSearch(final String queryString, final List<String> condQueries, final Identity identity, final Roles roles, final int firstResult,
        final int maxResults, final boolean doHighlighting) throws ServiceNotAvailableException, QueryException {

    synchronized (createIndexSearcherLock) {// o_clusterOK by:fj if service is only configured on one vm, which is recommended way
        if (searcher == null) {
            log.warn("Index does not exist, can't search for queryString: " + queryString);
            throw new ServiceNotAvailableException("Index does not exist");
        }
    }

    try {
        log.info("queryString=" + queryString);

        final BooleanQuery query = new BooleanQuery();
        if (StringHelper.containsNonWhitespace(queryString)) {
            final QueryParser queryParser = new MultiFieldQueryParser(Version.LUCENE_30, fields, analyzer);
            queryParser.setLowercaseExpandedTerms(false);// some add. fields are not tokenized and not lowered case
            final Query multiFieldQuery = queryParser.parse(queryString.toLowerCase());
            query.add(multiFieldQuery, Occur.MUST);
        }

        if (condQueries != null && !condQueries.isEmpty()) {
            for (final String condQueryString : condQueries) {
                final QueryParser condQueryParser = new QueryParser(Version.LUCENE_30, condQueryString, analyzer);
                condQueryParser.setLowercaseExpandedTerms(false);
                final Query condQuery = condQueryParser.parse(condQueryString);
                query.add(condQuery, Occur.MUST);
            }
        }

        if (log.isDebugEnabled()) {
            log.debug("query=" + query);
        }
        // TODO: 14.06.2010/cg : fellowig cide fragment can be removed later, do no longer call rewrite(query) because wildcard-search problem (OLAT-5359)
        // Query query = null;
        // try {
        // query = searcher.rewrite(query);
        // log.debug("after 'searcher.rewrite(query)' query=" + query);
        // } catch (Exception ex) {
        // throw new QueryException("Rewrite-Exception query because too many clauses. Query=" + query);
        // }
        final long startTime = System.currentTimeMillis();
        final int n = SearchServiceFactory.getService().getSearchModuleConfig().getMaxHits();
        final TopDocs docs = searcher.search(query, n);
        final long queryTime = System.currentTimeMillis() - startTime;
        if (log.isDebugEnabled()) {
            log.debug("hits.length()=" + docs.totalHits);
        }
        final SearchResultsImpl searchResult = new SearchResultsImpl(mainIndexer, searcher, docs, query, analyzer, identity, roles, firstResult, maxResults,
                doHighlighting);
        searchResult.setQueryTime(queryTime);
        searchResult.setNumberOfIndexDocuments(searcher.maxDoc());
        queryCount++;
        return searchResult;
    } catch (final ParseException pex) {
        throw new QueryException("can not parse query=" + queryString);
    } catch (final Exception ex) {
        log.warn("Exception in search", ex);
        throw new ServiceNotAvailableException(ex.getMessage());
    }
}
 

public static void main(String[] argv) {

        try {
            GlobalVar gv = GlobalVar.getGlobalVar();

            // get args
            File indexDir = gv.getIndexDir();
            File localDir = gv.getLocalDir();
            File root = gv.getLocalRoot();
            boolean hasWrappers = false;
            String usage = OfflineSearchIndexer.class.getName() + " [-wrappers]";
            for (int i = 0; i < argv.length; i++) {
                if (argv[i].equals("-wrappers")) { // parse -wrappers option
                    log.info("wrappers set true");
                    hasWrappers = true;
                } else {
                    log.error("Incorrect arguments in the command line");
                    System.err.println(usage);
                    System.err.println(" -wrappers means the directory contains wrappers saved in earlier run of seal");
                    return;
                }
            }

            // check args
            if (root!=null && !System.getenv("PWD").equals(root.getPath())) {
                log.error("to build an index relative to "+root+" run OfflineSearchIndexer from that directory, and make localDir a relative path");
                System.exit(-1);
            }
            if (root==null && !localDir.isAbsolute()) {
                log.warn("to build an absolute index make localDir an absolute path - this index will be relative to "+System.getenv("PWD"));
            }
            if (indexDir.exists()) {
                log.error("Cannot save index to '" +indexDir+ "' directory, please delete it first");
                System.exit(-1);
            }
            if (!localDir.exists() || !localDir.canRead()) {
                System.out.println("Document directory '" +localDir.getAbsolutePath()+ "' does not exist or is not readable, please check the path");
                System.exit(-1);
            }
            Date start = new Date();
            IndexWriter writer = new IndexWriter(FSDirectory.open(indexDir), new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.LIMITED);
            System.out.println("Indexing to directory '" +indexDir+ "'...");
            indexDocs(writer, localDir, hasWrappers);
            System.out.println("Optimizing...");
            writer.optimize();
            writer.close();

            Date end = new Date();
            log.info("indexed "+numIndexed+" of "+numFiles+" files");
            log.info((end.getTime() - start.getTime())+" total milliseconds");

        } catch (Exception e) {
            log.error(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
            e.printStackTrace();
        }
    }
 

/** @deprecated Use {@link #StandardFilter(Version, TokenStream)} instead. */
@Deprecated
public StandardFilter(final TokenStream in) {
  this(Version.LUCENE_30, in);
}