org.elasticsearch.index.analysis.NamedAnalyzer Java Examples

@Override
@SuppressWarnings("unchecked")
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    TokenCountFieldMapper.Builder builder = tokenCountField(name);
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String propName = Strings.toUnderscoreCase(entry.getKey());
        Object propNode = entry.getValue();
        if (propName.equals("null_value")) {
            builder.nullValue(nodeIntegerValue(propNode));
            iterator.remove();
        } else if (propName.equals("analyzer")) {
            NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
            if (analyzer == null) {
                throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
            }
            builder.analyzer(analyzer);
            iterator.remove();
        }
    }
    parseNumberField(builder, name, node, parserContext);
    if (builder.analyzer() == null) {
        throw new MapperParsingException("Analyzer must be set for field [" + name + "] but wasn't.");
    }
    return builder;
}
 

public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer)analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        final TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof ShingleTokenFilterFactory) {
                return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory();
            } else if (tokenFilterFactory instanceof ShingleTokenFilterFactory.Factory) {
                return (ShingleTokenFilterFactory.Factory) tokenFilterFactory;
            }
        }
    }
    return null;
}
 

public void testDefaultsIcuAnalysis() throws IOException {

        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY,
                new BundlePlugin(Settings.EMPTY));

        CharFilterFactory charFilterFactory = analysis.charFilter.get("icu_normalizer");
        assertThat(charFilterFactory, instanceOf(IcuNormalizerCharFilterFactory.class));

        TokenizerFactory tf = analysis.tokenizer.get("icu_tokenizer");
        assertThat(tf, instanceOf(IcuTokenizerFactory.class));

        TokenFilterFactory filterFactory = analysis.tokenFilter.get("icu_normalizer");
        assertThat(filterFactory, instanceOf(IcuNormalizerTokenFilterFactory.class));

        filterFactory = analysis.tokenFilter.get("icu_folding");
        assertThat(filterFactory, instanceOf(IcuFoldingTokenFilterFactory.class));

        filterFactory = analysis.tokenFilter.get("icu_transform");
        assertThat(filterFactory, instanceOf(IcuTransformTokenFilterFactory.class));

        Analyzer analyzer = analysis.indexAnalyzers.get( "icu_collation");
        assertThat(analyzer, instanceOf(NamedAnalyzer.class));
    }
 

@Override
public Mapper.Builder parse(final String name, final Map<String, Object> node,
        final ParserContext parserContext) throws MapperParsingException {
    final MinHashFieldMapper.Builder builder = new MinHashFieldMapper.Builder(
            name);
    parseField(builder, name, node, parserContext);
    for (final Iterator<Map.Entry<String, Object>> iterator = node.entrySet()
            .iterator(); iterator.hasNext();) {
        final Map.Entry<String, Object> entry = iterator.next();
        final String propName = entry.getKey();
        final Object propNode = entry.getValue();
        if (propName.equals("minhash_analyzer") && propNode != null) {
            final NamedAnalyzer analyzer = parserContext
                    .getIndexAnalyzers().get(propNode.toString());
            builder.minhashAnalyzer(analyzer);
            iterator.remove();
        } else if (propName.equals("copy_bits_to")
                && propNode != null) {
            parseCopyBitsFields(propNode, builder);
            iterator.remove();
        }
    }
    return builder;
}
 

@Override
public KeywordFieldMapper build(BuilderContext context) {
    setupFieldType(context);
    if (normalizerName != null) {
        NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName);
        if (normalizer == null) {
            throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
        }
        fieldType().setNormalizer(normalizer);
        final NamedAnalyzer searchAnalyzer;
        if (fieldType().splitQueriesOnWhitespace) {
            searchAnalyzer = indexAnalyzers.getWhitespaceNormalizer(normalizerName);
        } else {
            searchAnalyzer = normalizer;
        }
        fieldType().setSearchAnalyzer(searchAnalyzer);
    } else if (fieldType().splitQueriesOnWhitespace) {
        fieldType().setSearchAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()));
    }
    return new KeywordFieldMapper(
        name,
        position,
        defaultExpression,
        fieldType,
        defaultFieldType,
        ignoreAbove,
        lengthLimit,
        context.indexSettings(),
        multiFieldsBuilder.build(this, context),
        copyTo
    );
}
 

public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues,
		NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
		PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
		SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings,
		Map<String, FieldMapper<String>> fieldMappers,
		ThreadPool threadPool) {
	super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null,
			fieldDataSettings, indexSettings, multiFields, null);
	this.ontologySettings = oSettings;
	// Mappers are added to mappers map as they are used/created
	this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(indexSettings));
	mappers.mutator().putAll(fieldMappers).close();
	this.threadPool = threadPool;
}
 

public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues,
		NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
		PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
		SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings,
		Map<String, FieldMapper<String>> fieldMappers,
		ThreadPool threadPool) {
	super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null,
			fieldDataSettings, indexSettings, multiFields, null);
	this.ontologySettings = oSettings;
	// Mappers are added to mappers map as they are used/created
	this.mappers = CopyOnWriteHashMap.copyOf(fieldMappers);
	this.threadPool = threadPool;
}
 

public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues,
		NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
		PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
		SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings,
		Map<String, FieldMapper<String>> fieldMappers,
		ThreadPool threadPool) {
	super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null,
			fieldDataSettings, indexSettings, multiFields, null);
	this.ontologySettings = oSettings;
	// Mappers are added to mappers map as they are used/created
	this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(indexSettings));
	mappers.mutator().putAll(fieldMappers).close();
	this.threadPool = threadPool;
}
 

protected MinHashFieldMapper(final String simpleName, final MappedFieldType fieldType,
        final MappedFieldType defaultFieldType, final Settings indexSettings,
        final MultiFields multiFields, final CopyTo copyTo,
        final NamedAnalyzer minhashAnalyzer, final CopyBitsTo copyBitsTo) {
    super(simpleName, fieldType, defaultFieldType, indexSettings,
            multiFields, copyTo);
    this.minhashAnalyzer = minhashAnalyzer;
    this.copyBitsTo = copyBitsTo;
}
 

@Override
public StringFieldMapper build(BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    // if the field is not analyzed, then by default, we should omit norms and have docs only
    // index options, as probably what the user really wants
    // if they are set explicitly, we will use those values
    // we also change the values on the default field type so that toXContent emits what
    // differs from the defaults
    if (fieldType.indexOptions() != IndexOptions.NONE && !fieldType.tokenized()) {
        defaultFieldType.setOmitNorms(true);
        defaultFieldType.setIndexOptions(IndexOptions.DOCS);
        if (!omitNormsSet && fieldType.boost() == 1.0f) {
            fieldType.setOmitNorms(true);
        }
        if (!indexOptionsSet) {
            fieldType.setIndexOptions(IndexOptions.DOCS);
        }
    }
    setupFieldType(context);
    StringFieldMapper fieldMapper = new StringFieldMapper(
            name, fieldType, defaultFieldType, positionIncrementGap, ignoreAbove,
            context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
    return fieldMapper.includeInAll(includeInAll);
}
 

private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
    NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
    if (analyzer == null) {
        throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
    }
    return analyzer;
}
 

public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings indexSettings) {
    NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
    IndexAnalyzers indexAnalyzers =
            new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultAnalyzer, defaultAnalyzer, emptyMap(), emptyMap(), emptyMap());
    MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry();
    mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, mapperRegistry,
            () -> null);
}
 

@Override
public NamedAnalyzer indexAnalyzer() {
    return innerFieldType.indexAnalyzer();
}
 

@Override
public void normalizeOther(MappedFieldType other) {
    other.setSearchAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 

@Override
public void modify(MappedFieldType ft) {
    ft.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 

@Override
public void normalizeOther(MappedFieldType other) {
    other.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 

@Override
public void modify(MappedFieldType ft) {
    ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 

@Override
public void modify(MappedFieldType ft) {
    ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 

@Override
public void modify(MappedFieldType ft) {
    ft.setSearchQuoteAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 

@Override
public NamedAnalyzer searchQuoteAnalyzer() {
    return innerFieldType.searchQuoteAnalyzer();
}
 

@Override
public void normalizeOther(MappedFieldType other) {
    other.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 

void setAnalyzer(String name, Analyzer delegate) {
    setIndexAnalyzer(new NamedAnalyzer(name, AnalyzerScope.INDEX, new PhraseWrappedAnalyzer(delegate)));
}
 

@Override
public TextFieldMapper build(BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
                + name + "] without positions enabled");
        }
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    setupFieldType(context);
    PrefixFieldMapper prefixMapper = null;
    if (prefixFieldType != null) {
        if (fieldType().isSearchable() == false) {
            throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + name() + "]");
        }
        // Copy the index options of the main field to allow phrase queries on
        // the prefix field.
        if (context.indexCreatedVersion().onOrAfter(Version.ES_V_6_5_1)) {
            if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) {
                // frequencies are not needed because prefix queries always use a constant score
                prefixFieldType.setIndexOptions(IndexOptions.DOCS);
            } else {
                prefixFieldType.setIndexOptions(fieldType.indexOptions());
            }
        } else if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
            prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
        }
        if (fieldType.storeTermVectorOffsets()) {
            prefixFieldType.setStoreTermVectorOffsets(true);
        }
        prefixFieldType.setAnalyzer(fieldType.indexAnalyzer());
        prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings());
    }
    if (fieldType().indexPhrases) {
        if (fieldType().isSearchable() == false) {
            throw new IllegalArgumentException("Cannot set index_phrases on unindexed field [" + name() + "]");
        }
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set index_phrases on field [" + name() + "] if positions are not enabled");
        }
    }
    return new TextFieldMapper(
        name,
        position,
        defaultExpression,
        fieldType(),
        defaultFieldType,
        positionIncrementGap,
        prefixMapper,
        context.indexSettings(),
        multiFieldsBuilder.build(this, context),
        copyTo
    );
}
 

@Override
public void setSearchQuoteAnalyzer(NamedAnalyzer analyzer) {
    innerFieldType.setSearchQuoteAnalyzer(analyzer);
}
 

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
    String value;
    if (context.externalValueSet()) {
        value = context.externalValue().toString();
    } else {
        XContentParser parser = context.parser();
        if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
            value = fieldType().nullValueAsString();
        } else {
            value = parser.textOrNull();
        }
    }

    if (value == null || value.length() > ignoreAbove) {
        return;
    }

    final NamedAnalyzer normalizer = fieldType().normalizer();
    if (normalizer != null) {
        try (TokenStream ts = normalizer.tokenStream(name(), value)) {
            final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
            ts.reset();
            if (ts.incrementToken() == false) {
                throw new IllegalStateException("The normalization token stream is "
                    + "expected to produce exactly 1 token, but got 0 for analyzer " + normalizer
                    + " and input \"" + value + "\"");
            }
            final String newValue = termAtt.toString();
            if (ts.incrementToken()) {
                throw new IllegalStateException("The normalization token stream is "
                    + "expected to produce exactly 1 token, but got 2+ for analyzer " + normalizer
                    + " and input \"" + value + "\"");
            }
            ts.end();
            value = newValue;
        }
    }

    // convert to utf8 only once before feeding postings/dv/stored fields
    final BytesRef binaryValue = new BytesRef(value);
    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().name(), binaryValue, fieldType());
        fields.add(field);

        if (fieldType().hasDocValues() == false && fieldType().omitNorms()) {
            createFieldNamesField(context, fields);
        }
    }

    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
    }
}
 

@Override
public void setSearchAnalyzer(NamedAnalyzer analyzer) {
    innerFieldType.setSearchAnalyzer(analyzer);
}
 

@Override
public NamedAnalyzer searchAnalyzer() {
    return innerFieldType.searchAnalyzer();
}
 

@Override
public void setIndexAnalyzer(NamedAnalyzer analyzer) {
    innerFieldType.setIndexAnalyzer(analyzer);
}
 

@Override
public NamedAnalyzer indexAnalyzer() {
    return innerFieldType.indexAnalyzer();
}
 

@Override
public void normalizeOther(MappedFieldType other) {
    other.setSearchQuoteAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}