Java Code Examples for org.apache.lucene.analysis.util.CharArraySet#EMPTY_SET

public StandardAnalyzerProvider(Index index, Settings indexSettings, Environment env, String name, Settings settings) {
    super(index, indexSettings, name, settings);
    this.esVersion = Version.indexCreated(indexSettings);
    final CharArraySet defaultStopwords;
    if (esVersion.onOrAfter(Version.V_1_0_0_Beta1)) {
        defaultStopwords = CharArraySet.EMPTY_SET;
    } else {
        defaultStopwords = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
    }

    CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);
    int maxTokenLength = settings.getAsInt("max_token_length", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
    standardAnalyzer = new StandardAnalyzer(stopWords);
    standardAnalyzer.setVersion(version);
    standardAnalyzer.setMaxTokenLength(maxTokenLength);
}
 

@Inject
public PatternAnalyzerProvider(Index index, IndexSettingsService indexSettingsService, Environment env, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);

    Version esVersion = Version.indexCreated(indexSettingsService.getSettings());
    final CharArraySet defaultStopwords;
    if (esVersion.onOrAfter(Version.V_1_0_0_RC1)) {
        defaultStopwords = CharArraySet.EMPTY_SET;
    } else {
        defaultStopwords = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
    }
    boolean lowercase = settings.getAsBoolean("lowercase", true);
    CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);

    String sPattern = settings.get("pattern", "\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/);
    if (sPattern == null) {
        throw new IllegalArgumentException("Analyzer [" + name + "] of type pattern must have a `pattern` set");
    }
    Pattern pattern = Regex.compile(sPattern, settings.get("flags"));

    analyzer = new PatternAnalyzer(pattern, lowercase, stopWords);
}
 

public static CharArraySet parseStemExclusion(Settings settings, CharArraySet defaultStemExclusion) {
    String value = settings.get("stem_exclusion");
    if (value != null) {
        if ("_none_".equals(value)) {
            return CharArraySet.EMPTY_SET;
        } else {
            // LUCENE 4 UPGRADE: Should be settings.getAsBoolean("stem_exclusion_case", false)?
            return new CharArraySet(Strings.commaDelimitedListToSet(value), false);
        }
    }
    String[] stemExclusion = settings.getAsArray("stem_exclusion", null);
    if (stemExclusion != null) {
        // LUCENE 4 UPGRADE: Should be settings.getAsBoolean("stem_exclusion_case", false)?
        return new CharArraySet(Arrays.asList(stemExclusion), false);
    } else {
        return defaultStemExclusion;
    }
}
 

@Inject
public StandardHtmlStripAnalyzerProvider(Index index, IndexSettingsService indexSettingsService, Environment env,  @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);
    this.esVersion = Version.indexCreated(indexSettingsService.getSettings());
    final CharArraySet defaultStopwords;
    if (esVersion.onOrAfter(Version.V_1_0_0_RC1)) {
        defaultStopwords = CharArraySet.EMPTY_SET;
    } else {
        defaultStopwords = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
    }
    CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);
    analyzer = new StandardHtmlStripAnalyzer(stopWords);
    analyzer.setVersion(version);
}
 

@Inject
public SnowballAnalyzerProvider(Index index, IndexSettingsService indexSettingsService, Environment env, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);

    String language = settings.get("language", settings.get("name", "English"));
    CharArraySet defaultStopwords = defaultLanguageStopwords.containsKey(language) ? defaultLanguageStopwords.get(language) : CharArraySet.EMPTY_SET;
    CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);

    analyzer = new SnowballAnalyzer(language, stopWords);
    analyzer.setVersion(version);
}
 

public static CharArraySet parseWords(Environment env, Settings settings, String name, CharArraySet defaultWords, Map<String, Set<?>> namedWords, boolean ignoreCase) {
    String value = settings.get(name);
    if (value != null) {
        if ("_none_".equals(value)) {
            return CharArraySet.EMPTY_SET;
        } else {
            return resolveNamedWords(Strings.commaDelimitedListToSet(value), namedWords, ignoreCase);
        }
    }
    List<String> pathLoadedWords = getWordList(env, settings, name);
    if (pathLoadedWords != null) {
        return resolveNamedWords(pathLoadedWords, namedWords, ignoreCase);
    }
    return defaultWords;
}
 

@Override
public StandardAnalyzer getInstance(Map<String, String> params) throws IOException {
  if (params.containsKey(STOPWORDS_EMPTY)) {
    if (Boolean.parseBoolean(params.get(STOPWORDS_EMPTY))) {
      return new StandardAnalyzer(CharArraySet.EMPTY_SET);
    }
  }
  return new StandardAnalyzer();
}
 

@Nonnull
private static CharArraySet stopWords(@Nullable final String[] array)
        throws UDFArgumentException {
    if (array == null) {
        return SmartChineseAnalyzer.getDefaultStopSet();
    }
    if (array.length == 0) {
        return CharArraySet.EMPTY_SET;
    }
    CharArraySet results = new CharArraySet(Arrays.asList(array), true /* ignoreCase */);
    return results;
}
 

@Nonnull
private static CharArraySet stopWords(@Nullable final String[] array)
        throws UDFArgumentException {
    if (array == null) {
        return JapaneseAnalyzer.getDefaultStopSet();
    }
    if (array.length == 0) {
        return CharArraySet.EMPTY_SET;
    }
    return new CharArraySet(Arrays.asList(array), /* ignoreCase */true);
}
 

private static Object[] useEmptyStopWordSet(Object[] params) {
	Object rslt[] = new Object[params.length];
	for (int i=0; i<params.length; i++) {
		if (params[i] instanceof Set) {
			rslt[i] = CharArraySet.EMPTY_SET;
		} else {
			rslt[i] = params[i];
		}
	}
	return rslt;
}
 

public PatternAnalyzer(ConfigOptionsToAnalyzer lro, Pattern pattern, CharArraySet stopWords) throws Exception {
	/*
	super(lro.languageRange, getConstructor(PatternAnalyzerImpl.class,Pattern.class, CharArraySet.class), 
		pattern, stopWords);
		*/
	super(lro.languageRange, new PatternAnalyzerImpl(pattern, stopWords), new PatternAnalyzerImpl(pattern, CharArraySet.EMPTY_SET));
}
 

/**
 * This is called only when we have already identified that
 * the class does support stopwords.
 * @return
 */
public Set<?> getStopWords() {
	
	if (doNotUseStopWords()) 
		return CharArraySet.EMPTY_SET;
	
	if (useDefaultStopWords()) {
		return getStopWordsForClass(className);
	}
	
	return getStopWordsForClass(stopwords);
}
 

public PatternAnalyzer(ConfigOptionsToAnalyzer lro, Pattern pattern) throws Exception {
	super(lro.languageRange, new PatternAnalyzerImpl(pattern, CharArraySet.EMPTY_SET));
}
 

/**
 * The third and final phase of the life-cyle used for identifying
 * the AnalyzerPair.
 */
private AnalyzerPair construct() throws Exception {
	if (className == null) {
		return null;
	}
	if (pattern != null) {
		return new PatternAnalyzer(this, pattern);
	}
	if (softHyphens != null) {
		return new AnalyzerPair(
				languageRange,
				new TermCompletionAnalyzer(
						wordBoundary, 
						subWordBoundary, 
						softHyphens, 
						alwaysRemoveSoftHyphens));
	}
	if (wordBoundary != null) {
		return new AnalyzerPair(
				languageRange,
				new TermCompletionAnalyzer(
						wordBoundary, 
						subWordBoundary));
	}
	final Class<? extends Analyzer> cls = getAnalyzerClass();
          
          if (hasConstructor(cls, CharArraySet.class)) {
  			return new AnalyzerPair(languageRange,
  					cls.getConstructor(CharArraySet.class).newInstance(new CharArraySet(getStopWords(), false)),
  					cls.getConstructor(CharArraySet.class).newInstance(new CharArraySet(Collections.emptySet(), false)));
          }
          
          if (hasConstructor(cls, Version.class, Set.class)) {

          	// RussianAnalyzer is missing any way to access stop words.
          	if (RussianAnalyzer.class.equals(cls)) {
          		if (useDefaultStopWords()) {
          		    return new AnalyzerPair(languageRange, new RussianAnalyzer(), new RussianAnalyzer(CharArraySet.EMPTY_SET));
          		}
          		if (doNotUseStopWords()) {
          		    return new AnalyzerPair(languageRange,  new RussianAnalyzer(CharArraySet.EMPTY_SET));	
          		}
          	}
          	return new VersionSetAnalyzerPair(this, cls);
          }
          
          if (stopwords != null && !stopwords.equals(AnalyzerOptions.STOPWORDS_VALUE_NONE)) {
          	throw new RuntimeException("Bad option: language range: " + languageRange + " stopwords are not supported by " + className);
          }
          if (hasConstructor(cls, Version.class)) {
          	return new VersionAnalyzerPair(languageRange, cls);
          }
          
          if (hasConstructor(cls)) {
          	return new AnalyzerPair(languageRange, cls.newInstance());
          }
          throw new RuntimeException("Bad option: cannot find constructor for class " + className + " for language range " + languageRange);
}
 

/**
 * Returns the default stopwords set used by Lucene language analyzer for the specified language.
 *
 * @param language The language for which the stopwords are. The supported languages are English, French, Spanish,
 *                 Portuguese, Italian, Romanian, German, Dutch, Swedish, Norwegian, Danish, Russian, Finnish,
 *                 Irish, Hungarian, Turkish, Armenian, Basque and Catalan.
 * @return The default stopwords set used by Lucene language analyzers.
 */
private static CharArraySet getDefaultStopwords(String language) {
    switch (language) {
        case "English":
            return EnglishAnalyzer.getDefaultStopSet();
        case "French":
            return FrenchAnalyzer.getDefaultStopSet();
        case "Spanish":
            return SpanishAnalyzer.getDefaultStopSet();
        case "Portuguese":
            return PortugueseAnalyzer.getDefaultStopSet();
        case "Italian":
            return ItalianAnalyzer.getDefaultStopSet();
        case "Romanian":
            return RomanianAnalyzer.getDefaultStopSet();
        case "German":
            return GermanAnalyzer.getDefaultStopSet();
        case "Dutch":
            return DutchAnalyzer.getDefaultStopSet();
        case "Swedish":
            return SwedishAnalyzer.getDefaultStopSet();
        case "Norwegian":
            return NorwegianAnalyzer.getDefaultStopSet();
        case "Danish":
            return DanishAnalyzer.getDefaultStopSet();
        case "Russian":
            return RussianAnalyzer.getDefaultStopSet();
        case "Finnish":
            return FinnishAnalyzer.getDefaultStopSet();
        case "Irish":
            return IrishAnalyzer.getDefaultStopSet();
        case "Hungarian":
            return HungarianAnalyzer.getDefaultStopSet();
        case "Turkish":
            return SpanishAnalyzer.getDefaultStopSet();
        case "Armenian":
            return SpanishAnalyzer.getDefaultStopSet();
        case "Basque":
            return BasqueAnalyzer.getDefaultStopSet();
        case "Catalan":
            return CatalanAnalyzer.getDefaultStopSet();
        default:
            return CharArraySet.EMPTY_SET;
    }
}
 

public NoStopWordStandardAnalyzer() {
  super(LuceneVersionConstant.LUCENE_VERSION, CharArraySet.EMPTY_SET);
}