org.apache.lucene.search.spell.Dictionary Java Examples

@Test
public void testEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());
  
  IOUtils.close(ir, analyzer, dir);
}
 

@Test
public void testValueSourceEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, LongValuesSource.constant(10), PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());

  IOUtils.close(ir, analyzer, dir);
}
 

@Test
public void testLongValuesSourceEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, LongValuesSource.constant(10), PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());

  IOUtils.close(ir, analyzer, dir);
}
 

@Override
public Dictionary create(SolrCore core, SolrIndexSearcher searcher) {
  if (params == null) {
    // should not happen; implies setParams was not called
    throw new IllegalStateException("Value of params not set");
  }
  
  String sourceLocation = (String)params.get(Suggester.LOCATION);
  
  if (sourceLocation == null) {
    throw new IllegalArgumentException(Suggester.LOCATION + " parameter is mandatory for using FileDictionary");
  }
  
  String fieldDelimiter = (params.get(FIELD_DELIMITER) != null)
      ? (String) params.get(FIELD_DELIMITER) : 
      FileDictionary.DEFAULT_FIELD_DELIMITER;
  
  try {
    return new FileDictionary(new InputStreamReader(
        core.getResourceLoader().openResource(sourceLocation), StandardCharsets.UTF_8), fieldDelimiter);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public Dictionary create(SolrCore core, SolrIndexSearcher searcher) {
  if(params == null) {
    // should not happen; implies setParams was not called
    throw new IllegalStateException("Value of params not set");
  }
  String field = (String) params.get(FIELD);
  String weightField = (String) params.get(WEIGHT_FIELD);
  String payloadField = (String) params.get(PAYLOAD_FIELD);
  String contextField = (String) params.get(CONTEXT_FIELD);

  if (field == null) {
    throw new IllegalArgumentException(FIELD + " is a mandatory parameter");
  }

  return new DocumentDictionary(searcher.getIndexReader(), field, weightField, payloadField, contextField);
}
 

@Override
public Dictionary create(SolrCore core, SolrIndexSearcher searcher) {
  if(params == null) {
    // should not happen; implies setParams was not called
    throw new IllegalStateException("Value of params not set");
  }
  String field = (String)params.get(SolrSpellChecker.FIELD);
  
  if (field == null) {
    throw new IllegalArgumentException(SolrSpellChecker.FIELD + " is a mandatory parameter");
  }
  
  float threshold = params.get(THRESHOLD_TOKEN_FREQUENCY) == null ? 0.0f
      : (Float)params.get(THRESHOLD_TOKEN_FREQUENCY);
  
  return new HighFrequencyDictionary(searcher.getIndexReader(), field, threshold);
}
 

/**
  * Indexes the data from the given reader.
* @param reader Source index reader, from which autocomplete words are obtained for the defined field
* @param field the field of the source index reader to index for autocompletion
* @param mergeFactor mergeFactor to use when indexing
* @param ramMB the max amount or memory in MB to use
* @param optimize whether or not the autocomplete index should be optimized
  * @throws AlreadyClosedException if the Autocompleter is already closed
  * @throws IOException
  */
 public final void indexDictionary(IndexReader reader, String field, int mergeFactor, int ramMB, boolean optimize) throws IOException {
   synchronized (modifyCurrentIndexLock) {
     ensureOpen();
     final Directory dir = this.autoCompleteIndex;
     final Dictionary dict = new LuceneDictionary(reader, field);
     final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).setRAMBufferSizeMB(ramMB));
     IndexSearcher indexSearcher = obtainSearcher();
     final List<IndexReader> readers = new ArrayList<IndexReader>();

     if (searcher.maxDoc() > 0) {
       ReaderUtil.gatherSubReaders(readers, searcher.getIndexReader());
     }

     //clear the index
     writer.deleteAll();

     try {
       Iterator<String> iter = dict.getWordsIterator();

     while (iter.hasNext()) {
         String word = iter.next();

         // ok index the word
         Document doc = createDocument(word, reader.docFreq(new Term(field, word)));
         writer.addDocument(doc);
       }
     } finally {
       releaseSearcher(indexSearcher);
     }
     // close writer
     if (optimize)
       writer.optimize();
     writer.close();
     // also re-open the autocomplete index to see our own changes when the next suggestion
     // is fetched:
     swapSearcher(dir);
   }
 }
 

@Test
public void testWithOptionalPayload() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);

  // Create a document that is missing the payload field
  Document doc = new Document();
  Field field = new TextField(FIELD_NAME, "some field", Field.Store.YES);
  doc.add(field);
  // do not store the payload or the contexts
  Field weight = new NumericDocValuesField(WEIGHT_FIELD_NAME, 100);
  doc.add(weight);
  writer.addDocument(doc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);

  // Even though the payload field is missing, the dictionary iterator should not skip the document
  // because the payload field is optional.
  Dictionary dictionaryOptionalPayload =
      new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionaryOptionalPayload.getEntryIterator();
  BytesRef f = inputIterator.next();
  assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
  IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
  assertEquals(inputIterator.weight(), weightField.numericValue().longValue());
  IndexableField payloadField = doc.getField(PAYLOAD_FIELD_NAME);
  assertNull(payloadField);
  assertTrue(inputIterator.payload().length == 0);
  IOUtils.close(ir, analyzer, dir);
}
 

@Test
public void testMultiValuedField() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);

  List<Suggestion> suggestions = indexMultiValuedDocuments(atLeast(1000), writer);
  writer.commit();
  writer.close();

  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME, CONTEXT_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();
  BytesRef f;
  Iterator<Suggestion> suggestionsIter = suggestions.iterator();
  while((f = inputIterator.next())!=null) {
    Suggestion nextSuggestion = suggestionsIter.next();
    assertTrue(f.equals(nextSuggestion.term));
    long weight = nextSuggestion.weight;
    assertEquals(inputIterator.weight(), (weight != -1) ? weight : 0);
    assertEquals(inputIterator.payload(), nextSuggestion.payload);
    assertTrue(inputIterator.contexts().equals(nextSuggestion.contexts));
  }
  assertFalse(suggestionsIter.hasNext());
  IOUtils.close(ir, analyzer, dir);
}
 

public void testEmpty() throws Exception {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new HighFrequencyDictionary(ir, "bogus", 0.1f);
  BytesRefIterator tf = dictionary.getEntryIterator();
  assertNull(tf.next());
  dir.close();
}
 

@Override
public void indexingDone() {
	try {
		spellChecker = new DirectSpellChecker();
		spellChecker.setMaxEdits(2);
		spellChecker.setAccuracy(0.1f);
		spellChecker.setMinPrefix(0);
		reader = DirectoryReader.open(writer);

		fuzzySuggester = new FuzzySuggester(directory, "", writer.getAnalyzer());
		Dictionary dict = new DocumentValueSourceDictionary(reader, WORD_FIELD, new LongValuesSource() {
			
			@Override
			public boolean needsScores() {
				return false;
			}
			
			@Override
			public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
				return null;
			}
		});
		fuzzySuggester.build(dict);
		
		writer.close();
		searcher = new IndexSearcher(DirectoryReader.open(directory));
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

@Override
public Dictionary create(SolrCore core, SolrIndexSearcher searcher) {
  if(params == null) {
    // should not happen; implies setParams was not called
    throw new IllegalStateException("Value of params not set");
  }
  
  String field = (String) params.get(FIELD);
  String payloadField = (String) params.get(PAYLOAD_FIELD);
  String weightExpression = (String) params.get(WEIGHT_EXPRESSION);
  Set<SortField> sortFields = new HashSet<>();
  
  if (field == null) {
    throw new IllegalArgumentException(FIELD + " is a mandatory parameter");
  }
  
  if (weightExpression == null) {
    throw new IllegalArgumentException(WEIGHT_EXPRESSION + " is a mandatory parameter");
  }
  
  for(int i = 0; i < params.size(); i++) {
    if (params.getName(i).equals(SORT_FIELD)) {
      String sortFieldName = (String) params.getVal(i);

      sortFields.add(getSortField(core, sortFieldName));
    }
  }
 
  return new DocumentValueSourceDictionary(searcher.getIndexReader(), field, fromExpression(weightExpression,
      sortFields), payloadField);
}
 

@NotNull
private static SpellChecker createIndexSpellchecker(@NotNull Directory index) throws IOException {
    Directory spellCheckerDirectory = new RAMDirectory();
    IndexReader indexReader = DirectoryReader.open(index);
    Analyzer analyzer = new SimpleAnalyzer();
    IndexWriterConfig config = new IndexWriterConfig(analyzer);
    Dictionary dictionary = new HighFrequencyDictionary(indexReader, DRUG_TERMS_FIELD, 0.0f);
    SpellChecker spellChecker = new SpellChecker(spellCheckerDirectory);

    spellChecker.indexDictionary(dictionary, config, false);
    spellChecker.setAccuracy(SPELLCHECK_ACCURACY);
    return spellChecker;
}
 

/**
 * Creates a new spell-check index based on search-index
 */
public void createSpellIndex() {
    if (isSpellCheckEnabled) {
        IndexReader indexReader = null;
        try {
            log.info("Start generating Spell-Index...");
            long startSpellIndexTime = 0;
            if (log.isDebugEnabled()) {
                startSpellIndexTime = System.currentTimeMillis();
            }
            final Directory indexDir = FSDirectory.open(new File(indexPath));
            indexReader = IndexReader.open(indexDir);
            // 1. Create content spellIndex
            final File spellDictionaryFile = new File(spellDictionaryPath);
            final Directory contentSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + CONTENT_PATH));// true
            final SpellChecker contentSpellChecker = new SpellChecker(contentSpellIndexDirectory);
            final Dictionary contentDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.CONTENT_FIELD_NAME);
            contentSpellChecker.indexDictionary(contentDictionary);
            // 2. Create title spellIndex
            final Directory titleSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + TITLE_PATH));// true
            final SpellChecker titleSpellChecker = new SpellChecker(titleSpellIndexDirectory);
            final Dictionary titleDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.TITLE_FIELD_NAME);
            titleSpellChecker.indexDictionary(titleDictionary);
            // 3. Create description spellIndex
            final Directory descriptionSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + DESCRIPTION_PATH));// true
            final SpellChecker descriptionSpellChecker = new SpellChecker(descriptionSpellIndexDirectory);
            final Dictionary descriptionDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.DESCRIPTION_FIELD_NAME);
            descriptionSpellChecker.indexDictionary(descriptionDictionary);
            // 4. Create author spellIndex
            final Directory authorSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + AUTHOR_PATH));// true
            final SpellChecker authorSpellChecker = new SpellChecker(authorSpellIndexDirectory);
            final Dictionary authorDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.AUTHOR_FIELD_NAME);
            authorSpellChecker.indexDictionary(authorDictionary);

            // Merge all part spell indexes (content,title etc.) to one common spell index
            final Directory spellIndexDirectory = FSDirectory.open(spellDictionaryFile);// true
            final IndexWriter merger = new IndexWriter(spellIndexDirectory, new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
            final Directory[] directories = { contentSpellIndexDirectory, titleSpellIndexDirectory, descriptionSpellIndexDirectory, authorSpellIndexDirectory };
            merger.addIndexesNoOptimize(directories);
            merger.optimize();
            merger.close();
            spellChecker = new SpellChecker(spellIndexDirectory);
            spellChecker.setAccuracy(0.7f);
            if (log.isDebugEnabled()) {
                log.debug("SpellIndex created in " + (System.currentTimeMillis() - startSpellIndexTime) + "ms");
            }
            log.info("New generated Spell-Index ready to use.");
        } catch (final IOException ioEx) {
            log.warn("Can not create SpellIndex", ioEx);
        } finally {
            if (indexReader != null) {
                try {
                    indexReader.close();
                } catch (final IOException e) {
                    log.warn("Can not close indexReader properly", e);
                }
            }
        }
    }
}
 

public void updateSpellCheckerIndex(NodeDocumentVersion nDocVer) {
	log.info("Observed Wine added/updated event for {1} from Thread {0}",
			Thread.currentThread().getName(), String.valueOf(nDocVer));
	String text = (nDocVer != null) ? nDocVer.getText() : null;

	if (text != null) {
		Dictionary dictionary = null;

		try {
			FullTextEntityManager ftEm = (FullTextEntityManager) entityManager;
			SearchFactory searchFactory = ftEm.getSearchFactory();
			dictionary = new SetDictionary(text, searchFactory.getAnalyzer("wine_en"));
		} catch (IOException ioExc) {
			log.error("Failed to analyze dictionary text {0} from Wine {1} to update spell checker due to: {2}" +
					text + nDocVer.getUuid() + ioExc.toString());
		}

		if (dictionary != null) {
			Directory dir = null;
			// only allow one thread to update the index at a time ...
			// the Dictionary is pre-computed, so it should happen quickly
			// ...
			// this synchronized approach only works because this component
			// is application-scoped
			synchronized (this) {
				try {
					dir = FSDirectory.open(new File("lucene_index/spellcheck"));
					SpellChecker spell = new SpellChecker(dir);
					spell.indexDictionary(dictionary);
					spell.close();
					log.info("Successfully updated the spell checker index after Document added/updated.");
				} catch (Exception exc) {
					log.error("Failed to update the spell checker index!", exc);
				} finally {
					if (dir != null) {
						try {
							dir.close();
						} catch (Exception zzz) {
						}
					}
				}
			}
		}
	}
}
 

/**
 * Creates a new spell-check index based on search-index
 */
public static void createSpellIndex(final SearchModule searchModule) {
    final String tempSearchIndexPath = searchModule.getTempSearchIndexPath();
    final String tempSpellCheckIndexPath = searchModule.getTempSpellCheckerIndexPath();

    IndexReader indexReader = null;
    try {
        log.info("Start generating spell check index ...");

        long startSpellIndexTime = 0;
        if (log.isDebugEnabled()) {
            startSpellIndexTime = System.currentTimeMillis();
        }
        final Directory indexDir = FSDirectory.open(new File(tempSearchIndexPath, "main"));
        indexReader = IndexReader.open(indexDir);

        // 1. Create content spellIndex
        log.info("Generating 'content' spell check index ...");
        final File contentSpellIndexPath = new File(tempSpellCheckIndexPath + CONTENT_PATH);
        FileUtils.deleteDirsAndFiles(contentSpellIndexPath, true, true);
        final Directory contentSpellIndexDirectory = FSDirectory.open(contentSpellIndexPath);
        final SpellChecker contentSpellChecker = new SpellChecker(contentSpellIndexDirectory);
        final Dictionary contentDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.CONTENT_FIELD_NAME);
        contentSpellChecker.indexDictionary(contentDictionary);

        // 2. Create title spellIndex
        log.info("Generating 'title' spell check index ...");
        final File titleSpellIndexPath = new File(tempSpellCheckIndexPath + TITLE_PATH);
        FileUtils.deleteDirsAndFiles(titleSpellIndexPath, true, true);
        final Directory titleSpellIndexDirectory = FSDirectory.open(titleSpellIndexPath);
        final SpellChecker titleSpellChecker = new SpellChecker(titleSpellIndexDirectory);
        final Dictionary titleDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.TITLE_FIELD_NAME);
        titleSpellChecker.indexDictionary(titleDictionary);

        // 3. Create description spellIndex
        log.info("Generating 'description' spell check index ...");
        final File descriptionSpellIndexPath = new File(tempSpellCheckIndexPath + DESCRIPTION_PATH);
        FileUtils.deleteDirsAndFiles(descriptionSpellIndexPath, true, true);
        final Directory descriptionSpellIndexDirectory = FSDirectory.open(descriptionSpellIndexPath);
        final SpellChecker descriptionSpellChecker = new SpellChecker(descriptionSpellIndexDirectory);
        final Dictionary descriptionDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.DESCRIPTION_FIELD_NAME);
        descriptionSpellChecker.indexDictionary(descriptionDictionary);

        // 4. Create author spellIndex
        log.info("Generating 'author' spell check index ...");
        final File authorSpellIndexPath = new File(tempSpellCheckIndexPath + AUTHOR_PATH);
        FileUtils.deleteDirsAndFiles(authorSpellIndexPath, true, true);
        final Directory authorSpellIndexDirectory = FSDirectory.open(authorSpellIndexPath);
        final SpellChecker authorSpellChecker = new SpellChecker(authorSpellIndexDirectory);
        final Dictionary authorDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.AUTHOR_FIELD_NAME);
        authorSpellChecker.indexDictionary(authorDictionary);

        log.info("Merging spell check indices ...");
        // Merge all part spell indexes (content,title etc.) to one common spell index
        final File tempSpellCheckIndexDir = new File(tempSpellCheckIndexPath);
        FileUtils.deleteDirsAndFiles(tempSpellCheckIndexDir, true, true);
        final Directory tempSpellIndexDirectory = FSDirectory.open(tempSpellCheckIndexDir);
        final IndexWriter merger = new IndexWriter(tempSpellIndexDirectory, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
        final Directory[] directories = { contentSpellIndexDirectory, titleSpellIndexDirectory, descriptionSpellIndexDirectory, authorSpellIndexDirectory };
        merger.addIndexesNoOptimize(directories);

        log.info("Optimizing spell check index ...");
        merger.optimize();
        merger.close();

        tempSpellIndexDirectory.close();

        contentSpellChecker.close();
        contentSpellIndexDirectory.close();

        titleSpellChecker.close();
        titleSpellIndexDirectory.close();

        descriptionSpellChecker.close();
        descriptionSpellIndexDirectory.close();

        authorSpellChecker.close();
        authorSpellIndexDirectory.close();

        FileUtils.deleteDirsAndFiles(contentSpellIndexPath, true, true);
        FileUtils.deleteDirsAndFiles(titleSpellIndexPath, true, true);
        FileUtils.deleteDirsAndFiles(descriptionSpellIndexPath, true, true);
        FileUtils.deleteDirsAndFiles(authorSpellIndexPath, true, true);

        if (log.isDebugEnabled()) {
            log.debug("Spell check index created in " + (System.currentTimeMillis() - startSpellIndexTime) + " ms.");
        }
    } catch (final IOException ioEx) {
        log.warn("Can not create spell check index.", ioEx);
    } finally {
        if (indexReader != null) {
            try {
                indexReader.close();
            } catch (final IOException e) {
                log.warn("Can not close indexReader properly", e);
            }
        }
    }
}
 

/**
 * Create a Dictionary using options in <code>core</code> and optionally
 * uses <code>searcher</code>, in case of index based dictionaries
 */
public abstract Dictionary create(SolrCore core, SolrIndexSearcher searcher);
 

/** Build lookup from a dictionary. Some implementations may require sorted
 * or unsorted keys from the dictionary's iterator - use
 * {@link SortedInputIterator} or
 * {@link UnsortedInputIterator} in such case.
 */
public void build(Dictionary dict) throws IOException {
  build(dict.getEntryIterator());
}