org.elasticsearch.indices.analysis.AnalysisModule Java Examples

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
    Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> extra = new LinkedHashMap<>();
    if (settings.getAsBoolean("plugins.xbib.icu.enabled", true)) {
        extra.put("icu_collation", IcuCollationKeyAnalyzerProvider::new);
    }
    if (settings.getAsBoolean("plugins.xbib.hyphen.enabled", true)) {
        extra.put("hyphen", HyphenAnalyzerProvider::new);
    }
    if (settings.getAsBoolean("plugins.xbib.naturalsort.enabled", true)) {
        extra.put("naturalsort", NaturalSortKeyAnalyzerProvider::new);
    }
    if (settings.getAsBoolean("plugins.xbib.sortform.enabled", true)) {
        extra.put("sortform", SortformAnalyzerProvider::new);
    }
    if (settings.getAsBoolean("plugins.xbib.standardnumber.enabled", true)) {
        extra.put("standardnumber", (indexSettings, environment, name, factorySettings) ->
                new StandardnumberAnalyzerProvider(indexSettings, environment, name, factorySettings, standardNumberTypeParser));
    }
    return extra;
}
 

public static ESTestCase.TestAnalysis createTestAnalysisFromSettings(
        final Settings settings,
        final Path configPath,
        final AnalysisPlugin... plugins) throws IOException {
    final Settings actualSettings;
    if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) {
        actualSettings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
    } else {
        actualSettings = settings;
    }
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", actualSettings);
    final AnalysisRegistry analysisRegistry =
            new AnalysisModule(new Environment(actualSettings, configPath), Arrays.asList(plugins)).getAnalysisRegistry();
    return new ESTestCase.TestAnalysis(analysisRegistry.build(indexSettings),
            analysisRegistry.buildTokenFilterFactories(indexSettings),
            analysisRegistry.buildTokenizerFactories(indexSettings),
            analysisRegistry.buildCharFilterFactories(indexSettings));
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
    Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> extra = new HashMap<>();

    extra.put("hlseg_search", HLSegAnalyzerProvider::getHLSegSearchAnalyzerProvider);
    
    return extra;
}
 

/**
 * Returns a registered {@link Analyzer} provider by name or <code>null</code> if the analyzer was not registered
 */
public Analyzer getAnalyzer(String analyzer) throws IOException {
    AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer);
    if (analyzerProvider == null) {
        AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> provider = analyzers.get(analyzer);
        return provider == null ? null : cachedAnalyzer.computeIfAbsent(analyzer, (key) -> {
            try {
                return provider.get(environment, key).get();
            } catch (IOException ex) {
                throw new ElasticsearchException("failed to load analyzer for name " + key, ex);
            }
        });
    }
    return analyzerProvider.get(environment, analyzer).get();
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
    Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> extra = new LinkedHashMap<>();
    if (settings.getAsBoolean("plugins.xbib.icu.enabled", true)) {
        extra.put("icu_collation_tokenizer", IcuCollationTokenizerFactory::new);
        extra.put("icu_tokenizer", IcuTokenizerFactory::new);
    }
    if (settings.getAsBoolean("plugins.xbib.hyphen.enabled", true)) {
        extra.put("hyphen", HyphenTokenizerFactory::new);
    }
    if (settings.getAsBoolean("plugins.xbib.naturalsort.enabled", true)) {
        extra.put("naturalsort", NaturalSortKeyTokenizerFactory::new);
    }
    return extra;
}
 

public Map<String, TokenFilterFactory> buildTokenFilterFactories(IndexSettings indexSettings) throws IOException {
    final Map<String, Settings> tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER);
    Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters = new HashMap<>(this.tokenFilters);
    /*
     * synonym and synonym_graph are different than everything else since they need access to the tokenizer factories for the index.
     * instead of building the infrastructure for plugins we rather make it a real exception to not pollute the general interface and
     * hide internal data-structures as much as possible.
     */
    tokenFilters.put("synonym", requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
    tokenFilters.put("synonym_graph", requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings)));

    return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.preConfiguredTokenFilters);
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> getCharFilters() {
    Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> extra = new LinkedHashMap<>();
    if (settings.getAsBoolean("plugins.xbib.icu.enabled", true)) {
        extra.put("icu_normalizer", IcuNormalizerCharFilterFactory::new);
        extra.put("icu_folding", IcuFoldingCharFilterFactory::new);
    }
    return extra;
}
 

private static <T> AnalysisModule.AnalysisProvider<T> requiresAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
    return new AnalysisModule.AnalysisProvider<T>() {

        @Override
        public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
            return provider.get(indexSettings, environment, name, settings);
        }

        @Override
        public boolean requiresAnalysisSettings() {
            return true;
        }
    };
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.TokenFilterFactory>> getTokenFilters() {
    Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.TokenFilterFactory>> extra = new HashMap<>();
    extra.put("lc_full_pinyin", LcPinyinTokenFilterFactory::getLcFullPinyinTokenFilterFactory);
    extra.put("lc_first_letter", LcPinyinTokenFilterFactory::getLcFirstLetterTokenFilterFactory);
    return extra;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
    Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> providerMap
            = new HashMap<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>>();

    providerMap.put("lc_index", LcPinyinAnalyzerProvider::getIndexAnalyzerProvider);
    providerMap.put("lc_search", LcPinyinAnalyzerProvider::getSmartPinyinAnalyzerProvider);

    return providerMap;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
    Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> providerMap
            = new HashMap<String, AnalysisModule.AnalysisProvider<TokenizerFactory>>();

    providerMap.put("lc_index", LcPinyinTokenizerFactory::getLcIndexTokenizerFactory);
    providerMap.put("lc_search", LcPinyinTokenizerFactory::getLcSmartPinyinTokenizerFactory);

    return providerMap;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {

    //开启异步线程，先执行一个core的分词，试图去下载依赖的资源

    Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> map = new HashMap<>();

    map.put("mynlp", MynlpAnalyzerProvider::new);
    map.put("mynlp-core", MynlpAnalyzerProvider::new);
    if (enableCws) {
        map.put("mynlp-cws", MynlpAnalyzerProvider::new);
    }

    return map;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
    Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> extra = new HashMap<>(16);

    extra.put("mynlp", MynlpTokenizerFactory::new);
    extra.put("mynlp-core", MynlpTokenizerFactory::new);
    if (enableCws) {
        extra.put("mynlp-cws", MynlpTokenizerFactory::new);
    }

    return extra;
}
 

private <T> AnalysisModule.AnalysisProvider<T> requiresAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
    return new AnalysisModule.AnalysisProvider<T>() {

        @Override
        public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
            return provider.get(indexSettings, environment, name, settings);
        }

        @Override
        public boolean requiresAnalysisSettings() {
            return true;
        }
    };
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
    Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> analyzers = new HashMap<>();
    analyzers.put("hanlp", HanLPAnalyzerProvider::new);
    analyzers.put("hanlp-index", HanLPIndexAnalyzerProvider::new);
    return analyzers;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
    HashMap<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers = new HashMap<>();
    tokenizers.put("hanlp", HanLPTokenizerFactory::createStandard);
    tokenizers.put("hanlp-standard", HanLPTokenizerFactory::createStandard);
    tokenizers.put("hanlp-nlp", HanLPTokenizerFactory::createNLP);
    tokenizers.put("hanlp-index", HanLPIndexAnalyzerFactory::new);
    tokenizers.put("hanlp-nshort", HanLPTokenizerFactory::createNShort);
    tokenizers.put("hanlp-shortest", HanLPTokenizerFactory::createShortest);
    tokenizers.put("hanlp-crf", HanLPTokenizerFactory::createCRF);
    tokenizers.put("hanlp-speed", HanLPTokenizerFactory::createSpeed);
    return tokenizers;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
    Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> extra = new HashMap<>();

    extra.put("hanlp", HanLPAnalyzerProvider::getHanLPAnalyzerProvider);
    extra.put("hanlp_standard", HanLPAnalyzerProvider::getHanLPStandardAnalyzerProvider);
    extra.put("hanlp_index", HanLPAnalyzerProvider::getHanLPIndexAnalyzerProvider);
    extra.put("hanlp_nlp", HanLPAnalyzerProvider::getHanLPNLPAnalyzerProvider);
    extra.put("hanlp_n_short", HanLPAnalyzerProvider::getHanLPNShortAnalyzerProvider);
    extra.put("hanlp_dijkstra", HanLPAnalyzerProvider::getHanLPDijkstraAnalyzerProvider);
    extra.put("hanlp_crf", HanLPAnalyzerProvider::getHanLPCRFAnalyzerProvider);
    extra.put("hanlp_speed", HanLPAnalyzerProvider::getHanLPSpeedAnalyzerProvider);

    return extra;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
    Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();

    extra.put("hanlp", HanLPTokenizerFactory::getHanLPTokenizerFactory);
    extra.put("hanlp_standard", HanLPTokenizerFactory::getHanLPStandardTokenizerFactory);
    extra.put("hanlp_index", HanLPTokenizerFactory::getHanLPIndexTokenizerFactory);
    extra.put("hanlp_nlp", HanLPTokenizerFactory::getHanLPNLPTokenizerFactory);
    extra.put("hanlp_n_short", HanLPTokenizerFactory::getHanLPNShortTokenizerFactory);
    extra.put("hanlp_dijkstra", HanLPTokenizerFactory::getHanLPDijkstraTokenizerFactory);
    extra.put("hanlp_crf", HanLPTokenizerFactory::getHanLPCRFTokenizerFactory);
    extra.put("hanlp_speed", HanLPTokenizerFactory::getHanLPSpeedTokenizerFactory);

    return extra;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
    Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> extra = new HashMap<>();

    extra.put("ik_smart", IkAnalyzerProvider::getIkSmartAnalyzerProvider);
    extra.put("ik_max_word", IkAnalyzerProvider::getIkAnalyzerProvider);

    return extra;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
  Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> extra = new HashMap<>();

  extra.put("jieba_search", JiebaAnalyzerProvider::getJiebaSearchAnalyzerProvider);
  extra.put("jieba_index", JiebaAnalyzerProvider::getJiebaIndexAnalyzerProvider);

  return extra;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
    Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters = new HashMap<>();

    tokenFilters.put(PLUGIN_NAME, requiresAnalysisSettings((is, env, name, settings) -> new DynamicSynonymTokenFilterFactory(is, env, name, settings)));

    return tokenFilters;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
    Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();

    extra.put("hlseg_search", HLSegTokenizerFactory::getHLSegSearchTokenizerFactory);
   
    return extra;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
    return singletonMap("keyword", (indexSettings, environment, name, settings) -> {
        class Factory implements TokenizerFactory {

            @Override
            public Tokenizer create() {
                return new MockTokenizer(MockTokenizer.KEYWORD, false);
            }
        }
        return new Factory();
    });
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
  Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();

  extra.put("jieba_search", JiebaTokenizerFactory::getJiebaSearchTokenizerFactory);
  extra.put("jieba_index", JiebaTokenizerFactory::getJiebaIndexTokenizerFactory);

  return extra;
}
 

@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
    Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters = new HashMap<>();
    tokenFilters.put("openkoreantext-stemmer", OpenKoreanTextStemmerFactory::new);
    tokenFilters.put("openkoreantext-redundant-filter", OpenKoreanTextRedundantFilterFactory::new);
    tokenFilters.put("openkoreantext-phrase-extractor", OpenKoreanTextPhraseExtractorFactory::new);

    return tokenFilters;
}
 

/**
 * Creates an TestAnalysis with all the default analyzers configured.
 */
public static TestAnalysis createTestAnalysis(IndexSettings indexSettings, Settings nodeSettings,
                                              AnalysisPlugin... analysisPlugins) throws IOException {
    Environment env = TestEnvironment.newEnvironment(nodeSettings);
    AnalysisModule analysisModule = new AnalysisModule(env, Arrays.asList(analysisPlugins));
    AnalysisRegistry analysisRegistry = analysisModule.getAnalysisRegistry();
    return new TestAnalysis(analysisRegistry.build(indexSettings),
        analysisRegistry.buildTokenFilterFactories(indexSettings),
        analysisRegistry.buildTokenizerFactories(indexSettings),
        analysisRegistry.buildCharFilterFactories(indexSettings));
}
 

/**
 * Returns a registered {@link CharFilterFactory} provider by name or <code>null</code> if the char filter was not registered
 */
public AnalysisModule.AnalysisProvider<CharFilterFactory> getCharFilterProvider(String charFilter) {
    return charFilters.getOrDefault(charFilter, this.prebuiltAnalysis.getCharFilterFactory(charFilter));
}
 

private Builder(ClusterService clusterService,
                int numNodes,
                Random random,
                List<AnalysisPlugin> analysisPlugins) {
    if (numNodes < 1) {
        throw new IllegalArgumentException("Must have at least 1 node");
    }
    this.random = random;
    this.clusterService = clusterService;
    addNodesToClusterState(numNodes);
    functions = getFunctions();
    UserDefinedFunctionService udfService = new UserDefinedFunctionService(clusterService, functions);
    Map<String, SchemaInfo> schemaInfoByName = new HashMap<>();
    CrateSettings crateSettings = new CrateSettings(clusterService, clusterService.getSettings());
    schemaInfoByName.put("sys", new SysSchemaInfo(clusterService, crateSettings, new CeLicenseService()));
    schemaInfoByName.put("information_schema", new InformationSchemaInfo());
    schemaInfoByName.put(PgCatalogSchemaInfo.NAME, new PgCatalogSchemaInfo(udfService, tableStats));
    IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
    schemaInfoByName.put(
        BlobSchemaInfo.NAME,
        new BlobSchemaInfo(
            clusterService,
            new TestingBlobTableInfoFactory(
                Collections.emptyMap(), indexNameExpressionResolver, createTempDir())));


    Map<RelationName, DocTableInfo> docTables = new HashMap<>();
    DocTableInfoFactory tableInfoFactory = new TestingDocTableInfoFactory(
        docTables, functions, indexNameExpressionResolver);
    ViewInfoFactory testingViewInfoFactory = (ident, state) -> null;

    schemas = new Schemas(
        schemaInfoByName,
        clusterService,
        new DocSchemaInfoFactory(tableInfoFactory, testingViewInfoFactory, functions, udfService)
    );
    schemas.start();  // start listen to cluster state changes

    File homeDir = createTempDir();
    Environment environment = new Environment(
        Settings.builder().put(PATH_HOME_SETTING.getKey(), homeDir.getAbsolutePath()).build(),
        homeDir.toPath().resolve("config")
    );
    try {
        analysisRegistry = new AnalysisModule(environment, analysisPlugins).getAnalysisRegistry();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    fulltextAnalyzerResolver = new FulltextAnalyzerResolver(clusterService, analysisRegistry);
    createTableStatementAnalyzer = new CreateTableStatementAnalyzer(functions);
    createBlobTableAnalyzer = new CreateBlobTableAnalyzer(
        schemas,
        functions
    );
    allocationService = new AllocationService(
        new AllocationDeciders(
            Arrays.asList(
                new SameShardAllocationDecider(Settings.EMPTY, clusterService.getClusterSettings()),
                new ReplicaAfterPrimaryActiveAllocationDecider()
            )
        ),
        new TestGatewayAllocator(),
        new BalancedShardsAllocator(Settings.EMPTY),
        EmptyClusterInfoService.INSTANCE
    );

    publishInitialClusterState();
}
 

public AnalysisModule.AnalysisProvider<CharFilterFactory> getCharFilterFactory(String name) {
    return preConfiguredCharFilterFactories.get(name);
}
 

public AnalysisModule.AnalysisProvider<TokenFilterFactory> getTokenFilterFactory(String name) {
    return preConfiguredTokenFilters.get(name);
}