org.apache.lucene.analysis.standard.StandardAnalyzer Java Examples

@Test
public void shouldAddTermsFromSortOffset() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sort");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("offset")).thenReturn("100");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sort(anyString(), any(SortingParams.class))).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  final ArgumentCaptor<SortingParams> argument = ArgumentCaptor.forClass(SortingParams.class);
  verify(jedisMock).sort(eq("simpleKey"), argument.capture());
  Assert.assertEquals(getSortingParamString(new SortingParams().limit(100, 0)),
      getSortingParamString(argument.getValue()));
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Test
public void shouldDeflateGzipAndParseJsonTermsFromRedisOnGetCommand() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("get");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("compression")).thenReturn("gzip");
  when(localParamsMock.get("serialization")).thenReturn("json");

  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.get(any(byte[].class))).thenReturn(Compressor.compressGzip("[100,200,300]".getBytes()));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).get("simpleKey".getBytes());
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(3, terms.size());
}
 

public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
	String line = value.toString();

	CharArraySet noise = EnglishAnalyzer.getDefaultStopSet();
	// We are using a standard tokenizer that eliminates the stop words.
	// We can use Stemming tokenizer such Porter
	// A set of English noise keywords is used that will eliminates
	// words such as "the, a, etc"
	Analyzer analyzer = new StandardAnalyzer(noise);
	List<String> token = Tokenizer.tokenizeString(analyzer, line);
	Iterator<String> it = token.iterator();
	while (it.hasNext()) {
		word.set(it.next());
		fileName.set(key);
		if (!mapTable.containsKey(fileName.toString() + word.toString())) {
			context.write(fileName, word);
			mapTable.put(fileName.toString() + word.toString(), new IntWritable(1));
		}
	}
}
 

@Test
public void shouldAddTermsFromRedisOnZrangeCommandWithCustomRange() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("zrange");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("range_start")).thenReturn("1");
  when(localParamsMock.get("range_end")).thenReturn("100");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.zrangeWithScores(anyString(), anyLong(), anyLong()))
      .thenReturn(new HashSet<>(Arrays.asList(new Tuple("123", (double) 1.0f), new Tuple("321", (double) 1.0f))));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).zrangeWithScores("simpleKey", 1, 100);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Override
public boolean initialize(ResourceSpecifier aSpecifier, Map<String, Object> aAdditionalParams)
        throws ResourceInitializationException {
  super.initialize(aSpecifier, aAdditionalParams);
  hits = Integer.class.cast(getParameterValue("hits"));
  // query constructor
  String stoplistPath = String.class.cast(getParameterValue("stoplist-path"));
  try {
    stoplist = Resources.readLines(getClass().getResource(stoplistPath), UTF_8).stream()
            .map(String::trim).collect(toSet());
  } catch (IOException e) {
    throw new ResourceInitializationException(e);
  }
  analyzer = new StandardAnalyzer();
  parser = new QueryParser("text", analyzer);
  return true;
}
 

@Test
public void shouldAddTermsFromRedisOnLrangeCommandEmptyMinAndMaxFallsBackToDefault() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("lrange");
  when(localParamsMock.get("min")).thenReturn("");
  when(localParamsMock.get("max")).thenReturn("");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.lrange(anyString(), anyLong(), anyLong())).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).lrange("simpleKey", 0, -1);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Override
public void initialize(UimaContext context) throws ResourceInitializationException {
  super.initialize(context);
  hits = UimaContextHelper.getConfigParameterIntValue(context, "hits", 100);
  analyzer = UimaContextHelper.createObjectFromConfigParameter(context, "query-analyzer",
          "query-analyzer-params", StandardAnalyzer.class, Analyzer.class);
  queryStringConstructor = UimaContextHelper.createObjectFromConfigParameter(context,
          "query-string-constructor", "query-string-constructor-params",
          LuceneQueryStringConstructor.class, QueryStringConstructor.class);
  parser = new QueryParser("text", analyzer);
  // load parameters
  String param = UimaContextHelper.getConfigParameterStringValue(context, "doc-logreg-params");
  try {
    docFeatWeights = Resources.readLines(getClass().getResource(param), UTF_8).stream().limit(1)
            .map(line -> line.split("\t")).flatMap(Arrays::stream)
            .mapToDouble(Double::parseDouble).toArray();
  } catch (IOException e) {
    throw new ResourceInitializationException(e);
  }
}
 

@Transactional(readOnly = true)
public Integer createIndex(Integer siteId, Integer channelId,
		Date startDate, Date endDate, Integer startId, Integer max,
		Directory dir) throws IOException, ParseException {
	boolean exist = IndexReader.indexExists(dir);
	IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
			Version.LUCENE_30), !exist, IndexWriter.MaxFieldLength.LIMITED);
	try {
		if (exist) {
			LuceneContent.delete(siteId, channelId, startDate, endDate,
					writer);
		}
		Integer lastId = luceneContentDao.index(writer, siteId, channelId,
				startDate, endDate, startId, max);
		writer.optimize();
		return lastId;
	} finally {
		writer.close();
	}
}
 

@Test
public void shouldReturnEmptyQueryOnEmptyListOfSunion() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sunion");
  when(localParamsMock.get("key")).thenReturn("key1");
  when(localParamsMock.get("key1")).thenReturn("key2");
  when(localParamsMock.getParameterNamesIterator()).thenReturn(Arrays.asList("command", "key", "key1").iterator());
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sunion(anyString(), anyString())).thenReturn(new HashSet<String>());
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).sunion("key1", "key2");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(0, terms.size());
}
 

@Test
public void shouldReturnEmptyQueryOnEmptyListOfHget() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("hget");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("field")).thenReturn("f1");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.hget(anyString(), anyString())).thenReturn(null);
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).hget("simpleKey", "f1");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(0, terms.size());
}
 

@Test
public void shouldAddTermsFromSort() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sort");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sort(anyString(), any(SortingParams.class))).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  final ArgumentCaptor<SortingParams> argument = ArgumentCaptor.forClass(SortingParams.class);
  verify(jedisMock).sort(eq("simpleKey"), argument.capture());
  Assert.assertEquals(getSortingParamString(new SortingParams()), getSortingParamString(argument.getValue()));
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Test
public void shouldReturnEmptyQueryOnEmptyListOfSinter() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sinter");
  when(localParamsMock.get("key")).thenReturn("key1");
  when(localParamsMock.get("key1")).thenReturn("key2");
  when(localParamsMock.getParameterNamesIterator()).thenReturn(Arrays.asList("command", "key", "key1").iterator());
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sinter(anyString(), anyString())).thenReturn(new HashSet<String>());
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).sinter("key1", "key2");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(0, terms.size());
}
 

@Inject
public LucenePerUserWaveViewHandlerImpl(IndexDirectory directory,
                                        ReadableWaveletDataProvider waveletProvider,
                                        @Named(CoreSettingsNames.WAVE_SERVER_DOMAIN) String domain,
                                        @IndexExecutor Executor executor) {
  this.waveletProvider = waveletProvider;
  this.executor = executor;
  analyzer = new StandardAnalyzer(LUCENE_VERSION);
  try {
    IndexWriterConfig indexConfig = new IndexWriterConfig(LUCENE_VERSION, analyzer);
    indexConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
    indexWriter = new IndexWriter(directory.getDirectory(), indexConfig);
    nrtManager = new NRTManager(indexWriter, new WaveSearchWarmer(domain));
  } catch (IOException ex) {
    throw new IndexException(ex);
  }

  nrtManagerReopenThread = new NRTManagerReopenThread(nrtManager, MAX_STALE_SEC, MIN_STALE_SEC);
  nrtManagerReopenThread.start();
}
 

public static void main(String[] args) throws Exception {
    // index
    try (Directory index = new NIOFSDirectory(Paths.get("/tmp/index"))) {
        // add
        try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(new StandardAnalyzer()))) {
            Document doc = new Document();
            doc.add(new TextField("blog", "yuzhouwan.com", Field.Store.YES));
            doc.add(new StringField("github", "asdf2014", Field.Store.YES));
            writer.addDocument(doc);
            writer.commit();
        }
        // search
        try (DirectoryReader reader = DirectoryReader.open(index)) {
            IndexSearcher searcher = new IndexSearcher(reader);
            QueryParser parser = new QueryParser("blog", new StandardAnalyzer());
            Query query = parser.parse("yuzhouwan.com");
            ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
            for (ScoreDoc hit : hits) {
                Document hitDoc = searcher.doc(hit.doc);
                System.out.println(hitDoc.get("blog"));
            }
        }
    }
}
 

@Test
public void shouldAddTermsFromRedisOnLrangeCommandCustomMinAndMax() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("lrange");
  when(localParamsMock.get("min")).thenReturn("2");
  when(localParamsMock.get("max")).thenReturn("3");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.lrange(anyString(), anyLong(), anyLong())).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).lrange("simpleKey", 2, 3);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Test
public void shouldAddTermsFromRedisOnHmgetCommand() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("hmget");
  when(localParamsMock.get("key")).thenReturn("hash");
  when(localParamsMock.get("field")).thenReturn("field1");
  when(localParamsMock.getParameterNamesIterator()).thenReturn(Arrays.asList("command", "key", "field").iterator());
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.hmget(anyString(), anyString())).thenReturn(Arrays.asList("123"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).hmget("hash", "field1");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(1, terms.size());
}
 

@Test
public void shouldAddTermsFromSortOrderAsc() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sort");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("order")).thenReturn("asc");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sort(anyString(), any(SortingParams.class))).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  final ArgumentCaptor<SortingParams> argument = ArgumentCaptor.forClass(SortingParams.class);
  verify(jedisMock).sort(eq("simpleKey"), argument.capture());
  Assert.assertEquals(getSortingParamString(new SortingParams().asc()), getSortingParamString(argument.getValue()));
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

/**
 * Search for the user-specified query expression in the current page.
 * @throws Exception if an error occurs.
 */
private void search() throws Exception {
  final QueryParser parser = new QueryParser("contents", new StandardAnalyzer());
  final Query q = parser.parse(query);

  final MemoryIndex index = new MemoryIndex();
  final Link link = new Link(url);
  final PageData pageData = new SimpleHttpClientParser().load(link);
  index.addField("contents", pageData.getData().toString(), new StandardAnalyzer());
  final IndexSearcher searcher = index.createSearcher();
  final Hits hits = searcher.search(q);
  @SuppressWarnings("rawtypes")
  final Iterator it = hits.iterator();
  float relevance = 0f;
  if (it.hasNext()) {
    while (it.hasNext()) {
      final Hit hit = (Hit) it.next();
      relevance += ((float) Math.round(hit.getScore() * 1000)) / 10;
    }
    matchedLinks.add(new LinkMatch(url, relevance));
  }
}
 

@Override
String[] getExtraProperties() {
	String analyzer = ConfigurableAnalyzerFactory.Options.ANALYZER;
	return new String[]{
	FullTextIndex.Options.ANALYZER_FACTORY_CLASS, ConfigurableAnalyzerFactory.class.getName(),
	analyzer+"_."+AnalyzerOptions.LIKE, "x-empty",
	analyzer+"x-empty."+AnalyzerOptions.ANALYZER_CLASS, EmptyAnalyzer.class.getName(),
	analyzer+"x-terms."+AnalyzerOptions.PATTERN, "\\W+",
	analyzer+"x-splits."+AnalyzerOptions.ANALYZER_CLASS, TermCompletionAnalyzer.class.getName(),
	analyzer+"x-splits."+AnalyzerOptions.STOPWORDS, AnalyzerOptions.STOPWORDS_VALUE_NONE,
	analyzer+"x-splits."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-splits."+AnalyzerOptions.SUB_WORD_BOUNDARY, "(?<!\\p{L}|\\p{N})(?=\\p{L}|\\p{N})|(?<!\\p{Lu})(?=\\p{Lu})|(?<=\\p{N})(?=\\p{L})",
	analyzer+"x-hyphen."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "false",
	analyzer+"x-hyphen2."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen2."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen2."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen2."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "true",
	analyzer+"x-keywords."+AnalyzerOptions.ANALYZER_CLASS, KeywordAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.ANALYZER_CLASS, StandardAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.STOPWORDS, GermanAnalyzer.class.getName(),
	};
}
 

private void doMLTSearch() {
  if (Objects.isNull(mltDocFTF.getValue())) {
    throw new LukeException("Doc num is not set.");
  }
  int docNum = (int) mltDocFTF.getValue();
  MLTConfig mltConfig = operatorRegistry.get(MLTTabOperator.class)
      .map(MLTTabOperator::getConfig)
      .orElse(new MLTConfig.Builder().build());
  Analyzer analyzer = operatorRegistry.get(AnalysisTabOperator.class)
      .map(AnalysisTabOperator::getCurrentAnalyzer)
      .orElse(new StandardAnalyzer());
  Query query = searchModel.mltQuery(docNum, mltConfig, analyzer);
  Set<String> fieldsToLoad = operatorRegistry.get(FieldValuesTabOperator.class)
      .map(FieldValuesTabOperator::getFieldsToLoad)
      .orElse(Collections.emptySet());
  SearchResults results = searchModel.search(query, new SimilarityConfig.Builder().build(), fieldsToLoad, DEFAULT_PAGE_SIZE, false);

  TableUtils.setupTable(resultsTable, ListSelectionModel.SINGLE_SELECTION, new SearchResultsTableModel(), null,
      SearchResultsTableModel.Column.DOCID.getColumnWidth(),
      SearchResultsTableModel.Column.SCORE.getColumnWidth());
  populateResults(results);

  messageBroker.clearStatusMessage();
}
 

public static void main(String[] args) {

        String indexPath = "index";
        String docsPath = "/Users/admin/github/elasticsearch-full/JAVA-QUERY-API/";
        boolean create = true;

        final Path docDir = Paths.get(docsPath);
        if (!Files.isReadable(docDir)) {
            System.out.println("Document directory '" + docDir.toAbsolutePath() + "' does not exist or is not readable, please check the path");
            System.exit(1);
        }

        Date start = new Date();
        try {
            System.out.println("Indexing to directory '" + indexPath + "'...");
            Directory dir = FSDirectory.open(Paths.get(indexPath));
            Analyzer analyzer = new StandardAnalyzer();
            IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
            if (create) {
                iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
            } else {
                iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
            }

            IndexWriter writer = new IndexWriter(dir, iwc);
            indexDocs(writer, docDir);
            writer.close();
            Date end = new Date();
            System.out.println(end.getTime() - start.getTime() + " total milliseconds");
        } catch (Exception e) {
            System.out.println(" caught a " + e.getClass() +
                    "\n with message: " + e.getMessage());
        }

    }
 

public Index(final SearchModule searchModule, final MainIndexer mainIndexer, final long maxIndexTime, final String[] fields) {
    this.searchModule = searchModule;
    this.mainIndexer = mainIndexer;
    this.maxIndexTime = maxIndexTime;
    this.fields = fields;

    fullIndexer = new OlatFullIndexer(this, mainIndexer, searchModule);
    analyzer = new StandardAnalyzer(Version.LUCENE_30);

    createIndexSearcher(false);
    if (searchModule.isSpellCheckEnabled()) {
        createSpellCheckSearcher(false);
    }
}
 

@Test
public void givenTermQueryWhenFetchedDocumentThenCorrect() {
    InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
    inMemoryLuceneIndex.indexDocument("activity", "running in track");
    inMemoryLuceneIndex.indexDocument("activity", "Cars are running on road");

    Term term = new Term("body", "running");
    Query query = new TermQuery(term);

    List<Document> documents = inMemoryLuceneIndex.searchIndex(query);
    Assert.assertEquals(2, documents.size());
}
 

private void postInit() throws IOException {
	this.queryAnalyzer = new StandardAnalyzer();

	// do some initialization for new indices
	if (!DirectoryReader.indexExists(directory)) {
		logger.debug("creating new Lucene index in directory {}", directory);
		IndexWriterConfig indexWriterConfig = getIndexWriterConfig();
		indexWriterConfig.setOpenMode(OpenMode.CREATE);
		IndexWriter writer = new IndexWriter(directory, indexWriterConfig);
		writer.close();
	}
}
 

@Override
protected void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new StandardAnalyzer());

  FieldType textType = new FieldType();
  textType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  textType.setTokenized(true);
  textType.setStoreTermVectors(true);

  FieldType textType_pos = new FieldType();
  textType_pos.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  textType_pos.setTokenized(true);
  textType_pos.setStoreTermVectors(true);
  textType_pos.setStoreTermVectorPositions(true);

  FieldType textType_pos_offset = new FieldType();
  textType_pos_offset.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  textType_pos_offset.setTokenized(true);
  textType_pos_offset.setStoreTermVectors(true);
  textType_pos_offset.setStoreTermVectorPositions(true);
  textType_pos_offset.setStoreTermVectorOffsets(true);

  String text = "It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife.";
  Document doc = new Document();
  doc.add(newField("text1", text, textType));
  doc.add(newField("text2", text, textType_pos));
  doc.add(newField("text3", text, textType_pos_offset));
  writer.addDocument(doc);

  writer.commit();
  writer.close();
  dir.close();
}
 

@Test
public void shouldAddMultipleTermsFromRedisOnSrandmemberCommand() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("srandmember");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.srandmember(anyString(), anyInt())).thenReturn(Collections.singletonList("123"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).srandmember("simpleKey", 1);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(1, terms.size());
}
 

static boolean hasLuceneOperators(String query) throws ParseException {
    String sanitizedQueryForLucene = query.replaceAll("\\^(?!\\d)", "\\^1");
    org.apache.lucene.queryparser.classic.QueryParser parser =
            new org.apache.lucene.queryparser.classic.QueryParser("",
                    new StandardAnalyzer(new CharArraySet(0, false)));
    parser.setAllowLeadingWildcard(true);

    return hasOperator(parser.parse(sanitizedQueryForLucene));
}
 

public void testFreeFormQueryParse() throws Exception {
    String queryString = new String("name:spell -description:another");
    log.info("Original query: "  + queryString);

    NGramQueryParser parser = new NGramQueryParser("name",
            new NGramAnalyzer(min_ngram, max_ngram), true);
    Query q = parser.parse(queryString);
    log.info("NGramQueryParser parsed query:  " + q.toString());

    QueryParser origParser = new QueryParser("name", new StandardAnalyzer());
    q = origParser.parse(queryString);
    log.info("QueryParser parsed query = " + q.toString());
}
 

@Test
public void testParseClassic() {
  SearchImpl search = new SearchImpl(reader);
  QueryParserConfig config = new QueryParserConfig.Builder()
      .allowLeadingWildcard(true)
      .defaultOperator(QueryParserConfig.Operator.AND)
      .fuzzyMinSim(1.0f)
      .build();
  Query q = search.parseQuery("app~ f2:*ie", "f1", new StandardAnalyzer(),
      config, false);
  assertEquals("+f1:app~1 +f2:*ie", q.toString());
}
 

private Query parse(boolean rewrite) {
  String expr = StringUtils.isNullOrEmpty(queryStringTA.getText()) ? "*:*" : queryStringTA.getText();
  String df = operatorRegistry.get(QueryParserTabOperator.class)
      .map(QueryParserTabOperator::getDefaultField)
      .orElse("");
  QueryParserConfig config = operatorRegistry.get(QueryParserTabOperator.class)
      .map(QueryParserTabOperator::getConfig)
      .orElse(new QueryParserConfig.Builder().build());
  Analyzer analyzer = operatorRegistry.get(AnalysisTabOperator.class)
      .map(AnalysisTabOperator::getCurrentAnalyzer)
      .orElse(new StandardAnalyzer());
  return searchModel.parseQuery(expr, df, analyzer, config, rewrite);
}