com.hankcs.hanlp.seg.Segment Java Examples

public static void statistics(Segment segment, String inputFilePath) {
    try {
        //词频统计
        WordFreqStatistics statistic = new WordFreqStatistics(segment);
        BufferedReader reader = IOUtil.newBufferedReader(inputFilePath);
        String t;
        StringBuilder s = new StringBuilder();
        while ((t = reader.readLine()) != null) {
            s.append(t);
        }
        statistic.seg(s.toString());
        statistic.setResultPath(inputFilePath.replace(".txt", "") + "-WordFrequencyStatistics-Result.txt");
        statistic.dump();
        reader.close();
    } catch (IOException e) {
        logger.error("IO error: " + e.getLocalizedMessage());
    }
}
 

/**
 * 方法名： TranslatedName
 * 功 能： 取得音译人名识别
 * 参 数： @param str
 * 参 数： @return
 * 返 回： List<String>
 * 作 者 ： Tenghui.Wang
 * @throws
 */
public static List<String> TranslatedName(String[] str) {
	List<String> list = new ArrayList<String>();
	Segment segment = HanLP.newSegment().enableTranslatedNameRecognize(true);
	for (String sentence : str) {
		List<Term> termList = segment.seg(sentence);
		for (Term term : termList) {
			if (term.toString().contains("nrf")) {
				list.add(term.word);
			}
		}
	}
	return list;
}
 

/**
 * @param segment              HanLP中的某个分词器
 * @param filter               停用词
 * @param enablePorterStemming 英文原型转换
 */
public HanLpTokenizer(Segment segment, Set<String> filter) {
    super();
    this.segment = new SegmentWrapper(this.input, segment);
    if (filter != null && filter.size() > 0) {
        this.filter = new BinTrie<String>();
        for (String stopWord : filter) {
            this.filter.put(stopWord, null);
        }
    }
}
 

@Override
protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
    return new Analyzer.TokenStreamComponents(
        TokenizerBuilder.tokenizer(AccessController.doPrivileged((PrivilegedAction<Segment>)() -> {
            try {
                return new PerceptronLexicalAnalyzer();
            } catch (IOException e) {
                logger.error("can not use nlp analyzer, provider default", e);
                return HanLP.newSegment();
            }
        }), configuration));
}
 

/**
 * @param segment              HanLP中的某个分词器
 * @param filter               停用词
 * @param enablePorterStemming 英文原型转换
 */
public HanLPTokenizer(Segment segment, Set<String> filter, boolean enablePorterStemming) {
    super();
    this.segment = new SegmentWrapper(input, segment);
    if (filter != null && filter.size() > 0) {
        this.filter = new BinTrie<String>();
        for (String stopWord : filter) {
            this.filter.put(stopWord, null);
        }
    }
    this.enablePorterStemming = enablePorterStemming;
}
 

@Override
protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
    return new Analyzer.TokenStreamComponents(
        TokenizerBuilder.tokenizer(AccessController.doPrivileged((PrivilegedAction<Segment>)() -> {
            try {
                return new CRFLexicalAnalyzer();
            } catch (IOException e) {
                logger.error("can not use crf analyzer, provider default", e);
                return HanLP.newSegment();
            }
        }), configuration));
}
 

public static HanLPTokenizerFactory createCRF(IndexSettings indexSettings,
                                              Environment environment,
                                              String name,
                                              Settings settings) {
    return new HanLPTokenizerFactory(indexSettings, environment, name, settings) {
        @Override
        public Tokenizer create() {
            Segment seg = new CRFSegment().enablePartOfSpeechTagging(true);
            return new HanLPTokenizer(seg, defaultStopWordDictionary, enablePorterStemming);
        }
    };
}
 

public static HanLPTokenizerFactory createShortest(IndexSettings indexSettings,
                                                   Environment environment,
                                                   String name,
                                                   Settings settings) {
    return new HanLPTokenizerFactory(indexSettings, environment, name, settings) {
        @Override
        public Tokenizer create() {
            Segment seg = new DijkstraSegment().enableCustomDictionary(false)
                                               .enablePlaceRecognize(true)
                                               .enableOrganizationRecognize(true);
            return new HanLPTokenizer(seg, defaultStopWordDictionary, enablePorterStemming);
        }
    };
}
 

/**
 * 方法名： ChineseName
 * 功 能： 取得中文名
 * 参 数： @param str
 * 参 数： @return
 * 返 回： List<String>
 * 作 者 ： Tenghui.Wang
 * @throws
 */
public static List<String> ChineseName(String[] str) {
	List<String> list = new ArrayList<String>();
	Segment segment = HanLP.newSegment().enableNameRecognize(true);
	for (String sentence : str) {
		List<Term> termList = segment.seg(sentence);
		for (Term term : termList) {
			if (term.toString().contains("nr")) {
				list.add(term.word);
			}
		}
	}
	return list;
}
 

/**
 * 方法名： Organization
 * 功 能： 取得组织机构
 * 参 数： @param str
 * 参 数： @return
 * 返 回： List<String>
 * 作 者 ： Tenghui.Wang
 * @throws
 */
public static List<String> Organization(String[] str) {
	List<String> list = new ArrayList<String>();
	Segment segment = HanLP.newSegment().enableOrganizationRecognize(true);
	for (String sentence : str) {
		List<Term> termList = segment.seg(sentence);
		for (Term term : termList) {
			if (term.toString().contains("nt")) {
				list.add(term.word);
			}
		}
	}
	return list;
}
 

/**
 * 方法名： TranslatedName
 * 功 能： 取得日本人名
 * 参 数： @param str
 * 参 数： @return
 * 返 回： List<String>
 * 作 者 ： Tenghui.Wang
 * @throws
 */
public static List<String> JapaneseName(String[] str) {
	List<String> list = new ArrayList<String>();
	Segment segment = HanLP.newSegment().enableJapaneseNameRecognize(true);
	for (String sentence : str) {
		List<Term> termList = segment.seg(sentence);
		for (Term term : termList) {
			if (term.toString().contains("nrj")) {
				list.add(term.word);
			}
		}
	}
	return list;
}
 

public static void fileSegment(Segment segment, String inputFilePath, String outputFilePath) {
    try {
        WordFreqStatistics.statistics(segment, inputFilePath);
        BufferedReader reader = IOUtil.newBufferedReader(inputFilePath);
        long allCount = 0;
        long lexCount = 0;
        long start = System.currentTimeMillis();
        String outPath = inputFilePath.replace(".txt", "") + "-Segment-Result.txt";
        if (outputFilePath != null && outputFilePath.trim().length() > 0) outPath = outputFilePath;
        FileOutputStream fos = new FileOutputStream(new File(outPath));
        String temp;
        while ((temp = reader.readLine()) != null) {
            List<Term> parse = segment.seg(temp);
            StringBuilder sb = new StringBuilder();
            for (Term term : parse) {
                sb.append(term.toString() + "\t");
                if (term.word.trim().length() > 0) {
                    allCount += term.length();
                    lexCount += 1;
                }
            }
            fos.write(sb.toString().trim().getBytes());
            fos.write("\n".getBytes());
        }

        fos.flush();
        fos.close();
        reader.close();
        long end = System.currentTimeMillis();
        System.out.println("segment result save：" + outPath);
        System.out.println("total " + allCount + " chars, " + lexCount + " words, spend" + (end - start) + "ms ");
    } catch (IOException e) {
        logger.error("IO error: " + e.getLocalizedMessage());
    }
}
 

@Override
protected Tokenizer getSegmenter() {
    Segment segment = HanLP.newSegment();
    segment.enableOffset(true);
    HanLpTokenizer tokenizer = new HanLpTokenizer(segment, Collections.EMPTY_SET);
    return tokenizer;
}
 

/**
 * @param segment              HanLP中的某个分词器
 * @param filter               停用词
 * @param enablePorterStemming 英文原型转换
 */
public HanLPTokenizer(Segment segment, Set<String> filter, boolean enablePorterStemming)
{
    super();
    this.segment = new SegmentWrapper(input, segment);
    if (filter != null && filter.size() > 0)
    {
        this.filter = new BinTrie<String>();
        for (String stopWord : filter)
        {
            this.filter.put(stopWord, null);
        }
    }
    this.enablePorterStemming = enablePorterStemming;
}
 

@Override
public Segment build(Map<String, String> configurations) {
    String algorithm = get(configurations, "algorithm", "viterbi");
    Segment segment = HanLP.newSegment(algorithm);

    // 设置模式
    segment.enableIndexMode(getBoolean(configurations, "enableIndexMode", false));

    segment.enableOffset(true);

    // 是否识别数词和量词
    segment.enableNumberQuantifierRecognize(getBoolean(configurations, "enableNumberQuantifierRecognize", false));

    // 是否识别人名
    segment.enableNameRecognize(getBoolean(configurations, "enableNameRecognize", false));

    // 是否识别音译名
    // TODO 考虑是否依赖enableNameRecognize
    segment.enableTranslatedNameRecognize(getBoolean(configurations, "enableTranslatedNameRecognize", false));

    // 是否识别日本名?
    // TODO 考虑是否依赖enableNameRecognize
    segment.enableJapaneseNameRecognize(getBoolean(configurations, "enableJapaneseNameRecognize", false));

    // 是否识别组织名
    segment.enableOrganizationRecognize(getBoolean(configurations, "enableOrganizationRecognize", false));

    // 是否识别地名
    segment.enablePlaceRecognize(getBoolean(configurations, "enablePlaceRecognize", false));
    return segment;
}
 

public static HanLPTokenizerFactory createNShort(IndexSettings indexSettings,
                                                 Environment environment,
                                                 String name,
                                                 Settings settings) {
    return new HanLPTokenizerFactory(indexSettings, environment, name, settings) {
        @Override
        public Tokenizer create() {
            Segment seg = new NShortSegment().enableCustomDictionary(false)
                                             .enablePlaceRecognize(true)
                                             .enableOrganizationRecognize(true);
            return new HanLPTokenizer(seg, defaultStopWordDictionary, enablePorterStemming);
        }
    };
}
 

public WordFreqStatistics(String resultPath, Segment segment) {
    this.resultPath = resultPath;
    this.segment = segment;
}
 

public SegmentWrapper(Reader reader, Segment segment) {
    scanner = createScanner(reader);
    this.segment = segment;
}
 

public Segment getSegment() {
    return segment;
}
 

public void setSegment(Segment segment) {
    this.segment = segment;
}
 

public WordFreqStatistics(Segment segment) {
    this.segment = segment;
}
 

@Override
protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
    return new Analyzer.TokenStreamComponents(TokenizerBuilder.tokenizer(AccessController
            .doPrivileged((PrivilegedAction<Segment>)() -> new DoubleArrayTrieSegment().enableCustomDictionary(false)),
        configuration));
}
 

public SegmentWrapper(Reader reader, Segment segment)
{
    this.input = reader;
    this.segment = segment;
}
 

@Override
protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
    return new Analyzer.TokenStreamComponents(TokenizerBuilder.tokenizer(AccessController.doPrivileged((PrivilegedAction<Segment>)HanLP::newSegment), configuration));
}
 

@Override
protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
    return new Analyzer.TokenStreamComponents(TokenizerBuilder.tokenizer(
        AccessController.doPrivileged((PrivilegedAction<Segment>)() -> new NShortSegment().enableCustomDictionary(
            false).enablePlaceRecognize(true).enableOrganizationRecognize(true)), configuration));
}
 

@Override
protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
    return new Analyzer.TokenStreamComponents(TokenizerBuilder.tokenizer(AccessController.doPrivileged((PrivilegedAction<Segment>)HanLP::newSegment), configuration));
}
 

@Override
protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
    return new Analyzer.TokenStreamComponents(
        TokenizerBuilder.tokenizer(AccessController.doPrivileged((PrivilegedAction<Segment>)() ->
            HanLP.newSegment().enableIndexMode(true)), configuration));
}
 

public SegmentWrapper(Reader reader, Segment segment) {
    scanner = createScanner(reader);
    this.segment = segment;
}
 

public SegmentWrapper(Reader reader, Segment segment, Configuration configuration) {
    scanner = createScanner(reader);
    this.segment = segment;
    this.configuration = configuration;
}
 

/**
 * @param segment       HanLP中的某个分词器
 * @param configuration 分词配置
 */
public HanLPTokenizer(Segment segment, Configuration configuration) {
    this.configuration = configuration;
    this.segment = new SegmentWrapper(this.input, segment, configuration);
}