org.apache.lucene.queryParser.QueryParser Java Examples

/**
 * Search for the user-specified query expression in the current page.
 * @throws Exception if an error occurs.
 */
private void search() throws Exception {
  final QueryParser parser = new QueryParser("contents", new StandardAnalyzer());
  final Query q = parser.parse(query);

  final MemoryIndex index = new MemoryIndex();
  final Link link = new Link(url);
  final PageData pageData = new SimpleHttpClientParser().load(link);
  index.addField("contents", pageData.getData().toString(), new StandardAnalyzer());
  final IndexSearcher searcher = index.createSearcher();
  final Hits hits = searcher.search(q);
  @SuppressWarnings("rawtypes")
  final Iterator it = hits.iterator();
  float relevance = 0f;
  if (it.hasNext()) {
    while (it.hasNext()) {
      final Hit hit = (Hit) it.next();
      relevance += ((float) Math.round(hit.getScore() * 1000)) / 10;
    }
    matchedLinks.add(new LinkMatch(url, relevance));
  }
}
 

/**
 * Test searching with Lucene.
 * @param search the Lucene query text.
 * @param max the maximum number of results to show.
 * @throws Exception if an error is thrown while executing.
 */
public static void luceneSearch(final String search, final int max) throws Exception {
  print("Searching for: " + search);
  print("  max results: " + max);

  final IndexSearcher is = new IndexSearcher(index);
  final QueryParser parser = new QueryParser("contents", new StandardAnalyzer());

  final Query query = parser.parse(search);
  final Hits hits = is.search(query);

  print("    results: " + hits.length());

  for (int i = 0; i < Math.min(hits.length(), max); i++) {
    final float relevance = ((float) Math.round(hits.score(i) * 1000)) / 10;
    final String url = hits.doc(i).getField("url").stringValue();
    print("No " + (i + 1) + " with relevance " + relevance + "% : " + url);
  }

  is.close();
}
 

static int freq(Set<String> anchors, IndexSearcher index, QueryParser queryParser) throws IOException
{
	//int sum = 0;
	BitSet bits = new BitSet(index.maxDoc());
	for(String a : anchors)
	{
		try {
			Query q = queryParser.parse(String.format(QUERY_PATTERN, QueryParser.escape(a)));
			
			TotalHitCountCollectorSet results = new TotalHitCountCollectorSet(bits);
			
			index.search(q, results);
		
			//sum += results.getTotalHits();
		
		} catch (ParseException e) {
			
		}
	}
	return bits.cardinality();
}
 

public void testFreeFormQueryParse() throws Exception {
    String queryString = new String("name:spell -description:another");
    log.info("Original query: "  + queryString);

    NGramQueryParser parser = new NGramQueryParser("name",
            new NGramAnalyzer(min_ngram, max_ngram), true);
    Query q = parser.parse(queryString);
    log.info("NGramQueryParser parsed query:  " + q.toString());

    QueryParser origParser = new QueryParser("name", new StandardAnalyzer());
    q = origParser.parse(queryString);
    log.info("QueryParser parsed query = " + q.toString());
}
 

public Hits performSearch(Directory dir, Analyzer alyz, String query) throws Exception {
    QueryParser parser = new QueryParser("name", alyz);
    IndexSearcher searcher = new IndexSearcher(dir);
    Query q = parser.parse(query);
    Hits hits = searcher.search(q);
    return hits;
}
 

public void testFreeFormQueryParse() throws Exception {
    String queryString = new String("name:spell -description:another");
    log.info("Original query: "  + queryString);

    NGramQueryParser parser = new NGramQueryParser("name",
            new NGramAnalyzer(min_ngram, max_ngram), true);
    Query q = parser.parse(queryString);
    log.info("NGramQueryParser parsed query:  " + q.toString());

    QueryParser origParser = new QueryParser("name", new StandardAnalyzer());
    q = origParser.parse(queryString);
    log.info("QueryParser parsed query = " + q.toString());
}
 

public void testQueryParseWithSpecialChars() throws Exception {
    String queryString = new String("spell* virt- manager+");
    log.info("testQueryParserWithSpecialChars(): query string is: " + queryString);
    NGramQueryParser parser = new NGramQueryParser("name", new NGramAnalyzer(min_ngram, max_ngram));
    Query q = parser.parse(queryString);
    log.info("Using NGramQueryParser query = " + q.toString());

    QueryParser origParser = new QueryParser("name", new StandardAnalyzer());
    q = origParser.parse(queryString);
    log.info("Using QueryParser query = " + q.toString());
}
 

@Before
public void initializeLucene() throws IOException {
    directory = FSDirectory.open(new File(Main.LUCENE_INDEX));
    reader = IndexReader.open(directory, true);
    searcher = new IndexSearcher(reader);
    parser = new QueryParser(LuceneSearch.LUCENE_VERSION, getDefaultFieldName(), new StandardAnalyzer(LuceneSearch.LUCENE_VERSION));
}
 

private Query getBoboQuery( ) {
    Query boboQuery = luceneQueryNoFacetFilters;
    boolean hasNamespaceConstraint = ( namespaceConstraint != null && !namespaceConstraint.isEmpty( ) );
    boolean hasFieldConstraint = ( fieldConstraints != null && !fieldConstraints.isEmpty( ) );
    boolean hasRangeFacetSelection = (rangeFacetQuery != null && !rangeFacetQuery.clauses().isEmpty() );
    
    if ( hasNamespaceConstraint || hasFieldConstraint || hasRangeFacetSelection ) {
        //Apply field constraints and namespace constraints through query, when enabled
        BooleanQuery bq = new BooleanQuery( );
        bq.add( luceneQueryNoFacetFilters, Occur.MUST );

        if ( hasFieldConstraint ) {
            for ( Entry<String, String> kv : fieldConstraints.entrySet( ) ) {
                QueryParser qp = new QueryParser( LUCENEVERSION, kv.getKey( ), analyzer );
                try {
                    if ( allowLuceneSyntax )
                        bq.add( qp.parse( kv.getValue( ) ), Occur.MUST );
                    else
                        bq.add( qp.parse( QueryParser.escape( kv.getValue( ) ) ), Occur.MUST );
                } catch ( ParseException e ) {
                    Logger.error(e);
                }
            }
        }

        if ( hasNamespaceConstraint ) {
            bq.add( new TermQuery( new Term( SearchHelper.NAMESPACEFIELD, namespaceConstraint ) ), Occur.MUST );
        }
        if ( hasRangeFacetSelection ) {
            for( BooleanClause clause : rangeFacetQuery){
            	bq.add( clause );
            }
        }
        boboQuery = bq;
    }
    return boboQuery;
}
 

private final QueryParser getQueryParser( QueryDef qd ){
    QueryParser toReturn;
    if ( qd.boosts == null || qd.boosts.isEmpty( ) )
        toReturn = new SpecialMultiFieldQueryParser( LUCENEVERSION, qd.fields, analyzer );
    else
        toReturn = new SpecialMultiFieldQueryParser( LUCENEVERSION, qd.fields, analyzer, qd.boosts );
    toReturn.setDefaultOperator( defaultOperator );

    return toReturn;
}
 

/**
 * Perform search
 */
public static TopDocs performSearch(String field, String qs) throws IOException, ParseException {
	IndexSearcher searcher = Indexer.getIndexSearcher();
	QueryParser parser = new QueryParser(Config.LUCENE_VERSION, field, Indexer.getAnalyzer());
	Query query = parser.parse(qs);
	TopDocs result = searcher.search(query, Indexer.HITS_PER_PAGE);
	return result;
}
 

@Override
public ResultSet findByQueryPaginated(String token, String query, int offset, int limit) throws ParseException,
		AccessDeniedException, RepositoryException, DatabaseException {
	log.debug("findByQueryPaginated({}, {}, {}, {})", new Object[]{token, query, offset, limit});
	Authentication auth = null, oldAuth = null;

	try {
		if (token == null) {
			auth = PrincipalUtils.getAuthentication();
		} else {
			oldAuth = PrincipalUtils.getAuthentication();
			auth = PrincipalUtils.getAuthenticationByToken(token);
		}

		QueryParser qp = new QueryParser(Config.LUCENE_VERSION, "text", SearchDAO.analyzer);
		Query q = qp.parse(query);
		ResultSet rs = findByStatementPaginated(auth, q, offset, limit);
		log.debug("findByQueryPaginated: {}", rs);
		return rs;
	} catch (org.apache.lucene.queryParser.ParseException e) {
		throw new ParseException(e.getMessage(), e);
	} finally {
		if (token != null) {
			PrincipalUtils.setAuthentication(oldAuth);
		}
	}
}
 

protected Query createIndexQuery(String filedName, String searchStr) throws ParseException {
	Analyzer analyzer = AnalyzerFactory.createAnalyzer(searchStr);
	
    Query query1 =  new QueryParser(filedName, analyzer).parse(searchStr);
    //TermQuery query2 = new TermQuery(new Term(FIELD_ISSUEDATE, searchStr));
    TermQuery query3 = new TermQuery(new Term(FIELD_CREATETIME, searchStr));

    BooleanQuery booleanQuery = new BooleanQuery();
    booleanQuery.add(query1, BooleanClause.Occur.SHOULD);
    //booleanQuery.add(query2, BooleanClause.Occur.SHOULD);
    booleanQuery.add(query3, BooleanClause.Occur.SHOULD);
    return booleanQuery;
}
 

public Hits performSearch(Directory dir, Analyzer alyz, String query) throws Exception {
    QueryParser parser = new QueryParser("name", alyz);
    IndexSearcher searcher = new IndexSearcher(dir);
    Query q = parser.parse(query);
    Hits hits = searcher.search(q);
    return hits;
}
 

public void testQueryParseWithSpecialChars() throws Exception {
    String queryString = new String("spell* virt- manager+");
    log.info("testQueryParserWithSpecialChars(): query string is: " + queryString);
    NGramQueryParser parser = new NGramQueryParser("name", new NGramAnalyzer(min_ngram, max_ngram));
    Query q = parser.parse(queryString);
    log.info("Using NGramQueryParser query = " + q.toString());

    QueryParser origParser = new QueryParser("name", new StandardAnalyzer());
    q = origParser.parse(queryString);
    log.info("Using QueryParser query = " + q.toString());
}
 

public static String escapeQuery( String query ) {
    return QueryParser.escape( query );
}
 

private Query getParsedQuery( QueryDef qd ) throws ParseException {
   return new QueryParser( LUCENEVERSION, "", passThroughAnalyzer ).parse( qd.query );
}
 

@Override
	public void makeIndex(String lang, File workingDir) throws IOException
	{
		log.info("Loading support datasets...");
		
		File all_anchors = new WikipediaAnchorParser(lang).getFile();
		long numAnchors = ExternalSortUtils.wcl(all_anchors);
		AnchorIterator iterator = new AnchorIterator(all_anchors);
		
		IntSet people = new PeopleWIDs(lang).getDataset();
		
//		IndexSearcher articles = Indexes.getSearcher(RepositoryDirs.WIKIPEDIA.getPath(lang));
		IndexSearcher articles = openWikipediaIndex(lang);
		//QueryParser queryParser = new QueryParser(Version.LUCENE_34, WikipediaIndexer.FIELD_BODY, new WhitespaceAnalyzer(Version.LUCENE_34));
		QueryParser queryParser = new QueryParser(Version.LUCENE_34, WikipediaIndexer.FIELD_BODY, new StandardAnalyzer(Version.LUCENE_34, new HashSet<String>()));
		
		IndexWriter index = new IndexWriter(FSDirectory.open(workingDir.getAbsoluteFile()), new IndexWriterConfig(Version.LUCENE_34, new KeywordAnalyzer()));
		Document doc = new Document();
		Field fId = new Field(FIELD_ID, "", Store.YES, Index.NOT_ANALYZED);
		Field fText = new Field(FIELD_TEXT, "", Store.YES, Index.NOT_ANALYZED);
		Field fObject = new Field(FIELD_OBJECT, "", Store.YES, Index.NO);
		
		doc.add(fId);
		doc.add(fText);
		doc.add(fObject);
		
//		Field fOriginal = new Field(FIELD_ORIGINAL, "", Store.YES, Index.ANALYZED);
//		Field fWID = new Field(FIELD_WID, "", Store.NO, Index.ANALYZED);
		
		PLogger plog = new PLogger(log, Step.TEN_MINUTES, "lines", "anchors", "searches", "indexed", "0-freq","dropped");
		plog.setEnd(0, numAnchors);
		plog.start("Support datasets loaded, now parsing...");
		int id=0;
		while(iterator.next())
		{
			plog.update(0, iterator.scroll);
			plog.update(1);
			String anchorText = iterator.anchor;
			
			int freq = freq(iterator.originals, articles, queryParser);
			plog.update(2, iterator.originals.size());
			if (freq == 0) plog.update(4);
			
			Anchor anchorObj = Anchor.build(id, iterator.links, freq, people);
			if (anchorObj == null){
				plog.update(5);
				continue;
			}
			
			String anchorSerial = Anchor.serialize(anchorObj);
			fId.setValue(Integer.toString(++id));
			fText.setValue(anchorText);
			fObject.setValue(anchorSerial);
			
			for(int page : anchorObj){
				Field fWID = new Field(FIELD_WID, Integer.toString(page), Store.YES, Index.NOT_ANALYZED);
//				fWID.setBoost(iterator.links.get(page));
				doc.add(fWID);
			}
			for(String original : iterator.originals) {
				doc.add(new Field(FIELD_ORIGINAL, original, Store.YES, Index.NOT_ANALYZED));
			}
			
			index.addDocument(doc);
			plog.update(3);
			
			doc.removeFields(FIELD_ORIGINAL);
			doc.removeFields(FIELD_WID);
		}
		plog.stop();
		iterator.close();
		
		log.info("Now optimizing...");
		index.optimize();
		
		index.close();
		log.info("Done.");
	}
 

/**
 * Do search a certain query. The results will be filtered for the identity and roles.
 * 
 * @param queryString
 *            Search query-string.
 * @param identity
 *            Filter results for this identity (user).
 * @param roles
 *            Filter results for this roles (role of user).
 * @return SearchResults object for this query
 */
@Override
public SearchResults doSearch(final String queryString, final List<String> condQueries, final Identity identity, final Roles roles, final int firstResult,
        final int maxResults, final boolean doHighlighting) throws ServiceNotAvailableException, ParseException {
    try {
        if (!existIndex()) {
            log.warn("Index does not exist, can't search for queryString: " + queryString);
            throw new ServiceNotAvailableException("Index does not exist");
        }
        synchronized (createIndexSearcherLock) {// o_clusterOK by:fj if service is only configured on one vm, which is recommended way
            if (searcher == null) {
                try {
                    createIndexSearcher(indexPath);
                    checkIsIndexUpToDate();
                } catch (final IOException ioEx) {
                    log.warn("Can not create searcher", ioEx);
                    throw new ServiceNotAvailableException("Index is not available");
                }
            }
            if (hasNewerIndexFile()) {
                reopenIndexSearcher();
                checkIsIndexUpToDate();
            }
        }
        log.info("queryString=" + queryString);

        final BooleanQuery query = new BooleanQuery();
        if (StringHelper.containsNonWhitespace(queryString)) {
            final QueryParser queryParser = new MultiFieldQueryParser(Version.LUCENE_CURRENT, fields, analyzer);
            queryParser.setLowercaseExpandedTerms(false);// some add. fields are not tokenized and not lowered case
            final Query multiFieldQuery = queryParser.parse(queryString.toLowerCase());
            query.add(multiFieldQuery, Occur.MUST);
        }

        if (condQueries != null && !condQueries.isEmpty()) {
            for (final String condQueryString : condQueries) {
                final QueryParser condQueryParser = new QueryParser(Version.LUCENE_CURRENT, condQueryString, analyzer);
                condQueryParser.setLowercaseExpandedTerms(false);
                final Query condQuery = condQueryParser.parse(condQueryString);
                query.add(condQuery, Occur.MUST);
            }
        }

        if (log.isDebugEnabled()) {
            log.debug("query=" + query);
        }
        // TODO: 14.06.2010/cg : fellowig cide fragment can be removed later, do no longer call rewrite(query) because wildcard-search problem (OLAT-5359)
        // Query query = null;
        // try {
        // query = searcher.rewrite(query);
        // log.debug("after 'searcher.rewrite(query)' query=" + query);
        // } catch (Exception ex) {
        // throw new QueryException("Rewrite-Exception query because too many clauses. Query=" + query);
        // }
        final long startTime = System.currentTimeMillis();
        final int n = SearchServiceFactory.getService().getSearchModuleConfig().getMaxHits();
        final TopDocs docs = searcher.search(query, n);
        final long queryTime = System.currentTimeMillis() - startTime;
        if (log.isDebugEnabled()) {
            log.debug("hits.length()=" + docs.totalHits);
        }
        final SearchResultsImpl searchResult = new SearchResultsImpl(mainIndexer, searcher, docs, query, analyzer, identity, roles, firstResult, maxResults,
                doHighlighting);
        searchResult.setQueryTime(queryTime);
        searchResult.setNumberOfIndexDocuments(searcher.maxDoc());
        queryCount++;
        return searchResult;
    } catch (final ServiceNotAvailableException naex) {
        // pass exception
        throw new ServiceNotAvailableException(naex.getMessage());
    } catch (final ParseException pex) {
        throw new ParseException("can not parse query=" + queryString);
    } catch (final Exception ex) {
        log.warn("Exception in search", ex);
        throw new ServiceNotAvailableException(ex.getMessage());
    }
}
 

/**
 * @see org.olat.lms.search.SearchService#doSearch(String, List, Identity, Roles, int, int, boolean)
 */
public SearchResults doSearch(final String queryString, final List<String> condQueries, final Identity identity, final Roles roles, final int firstResult,
        final int maxResults, final boolean doHighlighting) throws ServiceNotAvailableException, QueryException {

    synchronized (createIndexSearcherLock) {// o_clusterOK by:fj if service is only configured on one vm, which is recommended way
        if (searcher == null) {
            log.warn("Index does not exist, can't search for queryString: " + queryString);
            throw new ServiceNotAvailableException("Index does not exist");
        }
    }

    try {
        log.info("queryString=" + queryString);

        final BooleanQuery query = new BooleanQuery();
        if (StringHelper.containsNonWhitespace(queryString)) {
            final QueryParser queryParser = new MultiFieldQueryParser(Version.LUCENE_30, fields, analyzer);
            queryParser.setLowercaseExpandedTerms(false);// some add. fields are not tokenized and not lowered case
            final Query multiFieldQuery = queryParser.parse(queryString.toLowerCase());
            query.add(multiFieldQuery, Occur.MUST);
        }

        if (condQueries != null && !condQueries.isEmpty()) {
            for (final String condQueryString : condQueries) {
                final QueryParser condQueryParser = new QueryParser(Version.LUCENE_30, condQueryString, analyzer);
                condQueryParser.setLowercaseExpandedTerms(false);
                final Query condQuery = condQueryParser.parse(condQueryString);
                query.add(condQuery, Occur.MUST);
            }
        }

        if (log.isDebugEnabled()) {
            log.debug("query=" + query);
        }
        // TODO: 14.06.2010/cg : fellowig cide fragment can be removed later, do no longer call rewrite(query) because wildcard-search problem (OLAT-5359)
        // Query query = null;
        // try {
        // query = searcher.rewrite(query);
        // log.debug("after 'searcher.rewrite(query)' query=" + query);
        // } catch (Exception ex) {
        // throw new QueryException("Rewrite-Exception query because too many clauses. Query=" + query);
        // }
        final long startTime = System.currentTimeMillis();
        final int n = SearchServiceFactory.getService().getSearchModuleConfig().getMaxHits();
        final TopDocs docs = searcher.search(query, n);
        final long queryTime = System.currentTimeMillis() - startTime;
        if (log.isDebugEnabled()) {
            log.debug("hits.length()=" + docs.totalHits);
        }
        final SearchResultsImpl searchResult = new SearchResultsImpl(mainIndexer, searcher, docs, query, analyzer, identity, roles, firstResult, maxResults,
                doHighlighting);
        searchResult.setQueryTime(queryTime);
        searchResult.setNumberOfIndexDocuments(searcher.maxDoc());
        queryCount++;
        return searchResult;
    } catch (final ParseException pex) {
        throw new QueryException("can not parse query=" + queryString);
    } catch (final Exception ex) {
        log.warn("Exception in search", ex);
        throw new ServiceNotAvailableException(ex.getMessage());
    }
}
 

public  ArrayList<String> searchType(String s, double thres1, double thres2, int k) throws Exception
{		
	Hits hits = null;
	String queryString = null;
	Query query = null;
	
	IndexSearcher searcher = new IndexSearcher(Globals.localPath+"data/DBpedia2016/lucene/type_fragment_index");
	
	ArrayList<String> typeNames = new ArrayList<String>(); 
	
	//String[] array = s.split(" ");
	//queryString = array[array.length-1];
	queryString = s;

	Analyzer analyzer = new StandardAnalyzer();
	try {
		QueryParser qp = new QueryParser("SplittedTypeShortName", analyzer);
		query = qp.parse(queryString);
	} catch (ParseException e) {
		e.printStackTrace();
	}
	
	if (searcher != null) {
		hits = searcher.search(query);
		
		System.out.println("find " + hits.length() + " answars!");
		if (hits.length() > 0) {
			for (int i=0; i<hits.length(); i++) {
				if (i < k) {
					System.out.println("<<<<---" + hits.doc(i).get("TypeShortName") + " : " + hits.score(i));
				    if(hits.score(i) >= thres1){
				    	System.out.println("Score>=thres1("+thres1+") ---" + hits.doc(i).get("TypeShortName") + " : " + hits.score(i));
				    	typeNames.add(hits.doc(i).get("TypeShortName"));
				    	//if (satisfiedStrictly(hits.doc(i).get("SplittedTypeShortName"), queryString)) typeNames.add(hits.doc(i).get("TypeShortName"));
				    }
				    else {
				    	//break;
				    }
				}
				else {
				    if(hits.score(i) >= thres2){
				    	System.out.println("<<<<---" + hits.doc(i).get("TypeShortName") + " : " + hits.score(i));
				    	typeNames.add(hits.doc(i).get("TypeShortName"));
				    	//if (satisfiedStrictly(hits.doc(i).get("SplittedTypeShortName"), queryString)) typeNames.add(hits.doc(i).get("TypeShortName"));
				    }
				    else {
				    	break;
				    }						
				}
			}				
		}
	}		
	return typeNames;	
}
 

public void testSearch() throws Exception {
   Query query = new QueryParser("contents",analyzer).parse("test");
 
    Hits hits = searcher.search(query);
 
    assertEquals("Find document(s)", 2, hits.length());
}
 

public void testSearch() throws Exception {
   Query query = QueryParser.parse("test", "contents", analyzer);
 
    Hits hits = searcher.search(query);
 
    assertEquals("Find document(s)", 2, hits.length());
}
 

/**
 * Creates the object and opens the index file.
 *
 * @param dictType
 *            the dictionary we will use for the search.
 * @param dictionaryPath
 *            overrides default dictionary location if non-null. An absolute
 *            os-specific path, e.g. /sdcard/aedict/index.
 * @param sort if true then the result list is always sorted.
 * @throws IOException
 *             on I/O error.
 */
public LuceneSearch(final DictTypeEnum dictType, final String dictionaryPath, final boolean sort) throws IOException {
    this.dictType = dictType;
    directory = FSDirectory.open(new File(dictionaryPath != null ? dictionaryPath : dictType.getDefaultDictionaryPath()));
    reader = IndexReader.open(directory, true);
    searcher = new IndexSearcher(reader);
    parser = new QueryParser(LUCENE_VERSION, "contents", new StandardAnalyzer(LUCENE_VERSION));
    this.sort = sort;
}