Python jieba.Tokenizer() Examples

The following are 14 code examples of jieba.Tokenizer(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module jieba , or try the search function .
Example #1
Source File: __init__.py    From jieba_fast with MIT License 5 votes vote down vote up 
def __init__(self, tokenizer=None):
        self.tokenizer = tokenizer or jieba.Tokenizer()
        self.load_word_tag(self.tokenizer.get_dict_file())
Example #2
Source File: __init__.py    From jieba_fast with MIT License 5 votes vote down vote up 
def lcut(self, *args, **kwargs):
        return list(self.cut(*args, **kwargs))

# default Tokenizer instance
Example #3
Source File: __init__.py    From chinese-support-redux with GNU General Public License v3.0 5 votes vote down vote up 
def __init__(self, tokenizer=None):
        self.tokenizer = tokenizer or jieba.Tokenizer()
        self.load_word_tag(self.tokenizer.get_dict_file())
Example #4
Source File: __init__.py    From chinese-support-redux with GNU General Public License v3.0 5 votes vote down vote up 
def lcut(self, *args, **kwargs):
        return list(self.cut(*args, **kwargs))

# default Tokenizer instance
Example #5
Source File: __init__.py    From Synonyms with MIT License 5 votes vote down vote up 
def __init__(self, tokenizer=None):
        self.tokenizer = tokenizer or jieba.Tokenizer()
        self.load_word_tag(self.tokenizer.get_dict_file())
Example #6
Source File: __init__.py    From Synonyms with MIT License 5 votes vote down vote up 
def lcut(self, *args, **kwargs):
        return list(self.cut(*args, **kwargs))

# default Tokenizer instance
Example #7
Source File: __init__.py    From QAbot_by_base_KG with MIT License 5 votes vote down vote up 
def __init__(self, tokenizer=None):
        self.tokenizer = tokenizer or jieba.Tokenizer()
        self.load_word_tag(self.tokenizer.get_dict_file())
Example #8
Source File: __init__.py    From QAbot_by_base_KG with MIT License 5 votes vote down vote up 
def lcut(self, *args, **kwargs):
        return list(self.cut(*args, **kwargs))

# default Tokenizer instance
Example #9
Source File: __init__.py    From python-girlfriend-mood with MIT License 5 votes vote down vote up 
def __init__(self, tokenizer=None):
        self.tokenizer = tokenizer or jieba.Tokenizer()
        self.load_word_tag(self.tokenizer.get_dict_file())
Example #10
Source File: __init__.py    From python-girlfriend-mood with MIT License 5 votes vote down vote up 
def lcut(self, *args, **kwargs):
        return list(self.cut(*args, **kwargs))

# default Tokenizer instance
Example #11
Source File: __init__.py    From annotated_jieba with MIT License 5 votes vote down vote up 
def __init__(self, tokenizer=None):
        self.tokenizer = tokenizer or jieba.Tokenizer()
        self.load_word_tag(self.tokenizer.get_abs_path_dict())
Example #12
Source File: __init__.py    From annotated_jieba with MIT License 5 votes vote down vote up 
def lcut(self, *args, **kwargs):
        return list(self.cut(*args, **kwargs))

# default Tokenizer instance
Example #13
Source File: __init__.py    From Malicious_Domain_Whois with GNU General Public License v3.0 5 votes vote down vote up 
def __init__(self, tokenizer=None):
        self.tokenizer = tokenizer or jieba.Tokenizer()
        self.load_word_tag(self.tokenizer.get_dict_file())
Example #14
Source File: __init__.py    From Malicious_Domain_Whois with GNU General Public License v3.0 5 votes vote down vote up 
def lcut(self, *args, **kwargs):
        return list(self.cut(*args, **kwargs))

# default Tokenizer instance