org.apache.lucene.analysis.path.PathHierarchyTokenizer Java Examples

@Inject
public PathHierarchyTokenizerFactory(Index index, IndexSettingsService indexSettingsService, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);
    bufferSize = settings.getAsInt("buffer_size", 1024);
    String delimiter = settings.get("delimiter");
    if (delimiter == null) {
        this.delimiter = PathHierarchyTokenizer.DEFAULT_DELIMITER;
    } else if (delimiter.length() > 1) {
        throw new IllegalArgumentException("delimiter can only be a one char value");
    } else {
        this.delimiter = delimiter.charAt(0);
    }

    String replacement = settings.get("replacement");
    if (replacement == null) {
        this.replacement = this.delimiter;
    } else if (replacement.length() > 1) {
        throw new IllegalArgumentException("replacement can only be a one char value");
    } else {
        this.replacement = replacement.charAt(0);
    }
    this.skip = settings.getAsInt("skip", PathHierarchyTokenizer.DEFAULT_SKIP);
    this.reverse = settings.getAsBoolean("reverse", false);
}
 

PathHierarchyTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
    super(indexSettings, name, settings);
    bufferSize = settings.getAsInt("buffer_size", 1024);
    String delimiter = settings.get("delimiter");
    if (delimiter == null) {
        this.delimiter = PathHierarchyTokenizer.DEFAULT_DELIMITER;
    } else if (delimiter.length() != 1) {
        throw new IllegalArgumentException("delimiter must be a one char value");
    } else {
        this.delimiter = delimiter.charAt(0);
    }

    String replacement = settings.get("replacement");
    if (replacement == null) {
        this.replacement = this.delimiter;
    } else if (replacement.length() != 1) {
        throw new IllegalArgumentException("replacement must be a one char value");
    } else {
        this.replacement = replacement.charAt(0);
    }
    this.skip = settings.getAsInt("skip", PathHierarchyTokenizer.DEFAULT_SKIP);
    this.reverse = settings.getAsBoolean("reverse", false);
}
 

@Override
public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
    List<PreConfiguredTokenizer> tokenizers = new ArrayList<>();
    tokenizers.add(PreConfiguredTokenizer.singleton("keyword", KeywordTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("classic", ClassicTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("uax_url_email", UAX29URLEmailTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("path_hierarchy", PathHierarchyTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("letter", LetterTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("whitespace", WhitespaceTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("ngram", NGramTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("edge_ngram",
        () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE), null));
    tokenizers.add(PreConfiguredTokenizer.singleton("pattern", () -> new PatternTokenizer(Regex.compile("\\W+", null), -1), null));
    tokenizers.add(PreConfiguredTokenizer.singleton("thai", ThaiTokenizer::new, null));
    tokenizers.add(PreConfiguredTokenizer.singleton("lowercase", XLowerCaseTokenizer::new, () -> new TokenFilterFactory() {
        @Override
        public String name() {
            return "lowercase";
        }

        @Override
        public TokenStream create(TokenStream tokenStream) {
            return new LowerCaseFilter(tokenStream);
        }
    }));

    // Temporary shim for aliases. TODO deprecate after they are moved
    tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new, null));

    return tokenizers;
}
 

@Override
public Tokenizer create() {
    if (reverse) {
        return new ReversePathHierarchyTokenizer(bufferSize, delimiter, replacement, skip);
    }
    return new PathHierarchyTokenizer(bufferSize, delimiter, replacement, skip);
}
 

private List<Token> getPathTokens(String url, String partStringRaw, String partString) throws IOException {
    int start = getStartIndex(url, partStringRaw);
    if (!tokenizePath) {
        int end = getEndIndex(start, partStringRaw);
        return Collections.singletonList(new Token(partString, URLPart.PATH, start, end));
    }
    return tokenize(URLPart.PATH, addReader(new PathHierarchyTokenizer('/', '/'), new StringReader(partString)), start);
}
 

@Override
public Tokenizer create() {
    if (reverse) {
        return new ReversePathHierarchyTokenizer(bufferSize, delimiter, replacement, skip);
    }
    return new PathHierarchyTokenizer(bufferSize, delimiter, replacement, skip);
}