Python nltk.stem.porter.PorterStemmer() Examples

def allign_words(hypothesis, reference, stemmer = PorterStemmer(), wordnet = wordnet):
    """
    Aligns/matches words in the hypothesis to reference by sequentially 
    applying exact match, stemmed match and wordnet based synonym match. 
    In case there are multiple matches the match which has the least number
    of crossing is chosen.

    :param hypothesis: hypothesis string
    :param reference: reference string
    :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer())
    :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method
    :param wordnet: a wordnet corpus reader object (default nltk.corpus.wordnet)
    :type wordnet: WordNetCorpusReader
    :return: sorted list of matched tuples, unmatched hypothesis list, unmatched reference list
    :rtype: list of tuples, list of tuples, list of tuples
    """
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _enum_allign_words(enum_hypothesis_list, enum_reference_list, stemmer= stemmer,
                             wordnet= wordnet) 

def load(tdb):
    # load the tasks and arxiv metadata
    stemmer = PorterStemmer()

    tdb.load_tasks("data/tasks/nlpprogress.json")
    tdb.load_synonyms(["data/tasks/synonyms.csv"])
    arxiv = serialization.load(
        "data/arxiv_aclweb.json.gz", fmt=serialization.Format.json_gz
    )

    for a in arxiv:
        if a["abstract"] is None:
            a["abstract"] = ""

    # require and normalise arxiv titles
    arxiv = [a for a in arxiv if "title" in a and a["title"] is not None]
    for a in arxiv:
        a["title"] = re.sub(" +", " ", a["title"].replace("\n", " "))
        a["title_lower"] = a["title"].lower()
        a["abstract_lower"] = a["abstract"].lower()
        a["title_stem"] = stemmer.stem(a["title"])
        a["abstract_stem"] = stemmer.stem(a["abstract"])

    return arxiv 

def stem_match(hypothesis, reference, stemmer = PorterStemmer()):
    """
    Stems each word and matches them in hypothesis and reference 
    and returns a word mapping between hypothesis and reference

    :param hypothesis:
    :type hypothesis:
    :param reference:
    :type reference:
    :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer())
    :type stemmer: nltk.stem.api.StemmerI or any class that 
                   implements a stem method
    :return: enumerated matched tuples, enumerated unmatched hypothesis tuples, 
             enumerated unmatched reference tuples
    :rtype: list of 2D tuples, list of 2D tuples,  list of 2D tuples
    """
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _enum_stem_match(enum_hypothesis_list, enum_reference_list, stemmer = stemmer) 

def __init__(self,paragraphs,removeStopWord = False,useStemmer = False):
        self.idf = {}               # dict to store IDF for words in paragraph
        self.paragraphInfo = {}     # structure to store paragraphVector
        self.paragraphs = paragraphs
        self.totalParas = len(paragraphs)
        self.stopwords = stopwords.words('english')
        self.removeStopWord = removeStopWord
        self.useStemmer = useStemmer
        self.vData = None
        self.stem = lambda k:k.lower()
        if(useStemmer):
            ps = PorterStemmer()
            self.stem = ps.stem
            
        # Initialize
        self.computeTFIDF()
        
    # Return term frequency for Paragraph
    # Input:
    #       paragraph(str): Paragraph as a whole in string format
    # Output:
    #       wordFrequence(dict) : Dictionary of word and term frequency 

def test_dictionary_match(doc_setup):
    """Test DictionaryMatch matcher."""
    doc = doc_setup
    space = MentionNgrams(n_min=1, n_max=1)

    # Test with a list of str
    matcher = DictionaryMatch(d=["this"])
    assert set(tc.get_span() for tc in matcher.apply(space.apply(doc))) == {"This"}

    # Test without a dictionary
    with pytest.raises(Exception):
        DictionaryMatch()

    # TODO: test with plural words
    matcher = DictionaryMatch(d=["is"], stemmer=PorterStemmer())
    assert set(tc.get_span() for tc in matcher.apply(space.apply(doc))) == {"is"}

    # Test if matcher raises an error when _f is given non-TemporarySpanMention
    matcher = DictionaryMatch(d=["this"])
    with pytest.raises(ValueError):
        list(matcher.apply(doc.sentences[0].words)) 

def tiny_tokenize(text, stem=False, stop_words=[]):
    words = []
    for token in wordpunct_tokenize(re.sub('[%s]' % re.escape(string.punctuation), ' ', \
            text.decode(encoding='UTF-8', errors='ignore'))):
        if not token.isdigit() and not token in stop_words:
            if stem:
                try:
                    w = EnglishStemmer().stem(token)
                except Exception as e:
                    w = token
            else:
                w = token
            words.append(w)

    return words

    # return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
    #                     re.sub('[%s]' % re.escape(string.punctuation), ' ', text.decode(encoding='UTF-8', errors='ignore'))) if
    #                     not token.isdigit() and not token in stop_words] 

def sim_sentence(self, queryVector, sentence):
        sentToken = word_tokenize(sentence)
        ps = PorterStemmer()
        for index in range(0,len(sentToken)):
            sentToken[index] = ps.stem(sentToken[index])
        sim = 0
        for word in queryVector.keys():
            w = ps.stem(word)
            if w in sentToken:
                sim += 1
        return sim/(len(sentToken)*len(queryVector.keys()))
    
    # Get Named Entity from the sentence in form of PERSON, GPE, & ORGANIZATION
    # Input:
    #       answers(list)       : List of potential sentence containing answer
    # Output:
    #       chunks(list)        : List of tuple with entity and name in ranked 
    #                             order 

def load_data(dname):
    stemmer = PorterStemmer()
    qids, questions, answers, labels = [], [], [], []
    print('Load folder ' + dname)
    with open(dname+'a.toks', encoding='utf-8') as f:
        for line in f:
            question = line.strip().split()
            question = [stemmer.stem(word) for word in question]
            questions.append(question)
    with open(dname+'b.toks', encoding='utf-8') as f:
        for line in f:
            answer = line.strip().split()
            answer_list = []
            for word in answer:
                try:
                    answer_list.append(stemmer.stem(word))
                except Exception as e:
                    print("couldn't stem the word:" + word)
            answers.append(answer_list)
    with open(dname+'id.txt', encoding='utf-8') as f:
        for line in f:
            qids.append(line.strip())
    with open(dname+'sim.txt', encoding='utf-8') as f:
        for line in f:
            labels.append(int(line.strip()))
    return qids, questions, answers, labels 

def get_stemmed_terms_list(doc, stem_words_map = None, stem_bigrams_map = None):
    ps = PorterStemmer()
    local_map = dict()
    word_list = []

    clean_doc = [(w.strip()).lower() for w in doc.split() if len(w) in range(3,16)]
    filtered_words = [w.strip('.,;?!:)(#') for w in clean_doc if not w.strip('.,;?!:)(#') in stopwords.words('english')]

    for w in filtered_words:
        if w.isalpha():
            w_temp = ps.stem_word(w)
            if stem_words_map is not None:
                if w_temp not in stem_words_map:
                    stem_words_map[w_temp] = dict()
                stem_words_map[w_temp][w] = stem_words_map[w_temp].get(w, 0)+1
                local_map[w_temp] = w
            word_list.append(w_temp)

    bigrams = nltk.bigrams(word_list)
    for b in bigrams:
        bigram_org = (local_map[b[0]],local_map[b[1]])
        if stem_bigrams_map is not None:
                if b not in stem_bigrams_map:
                    stem_bigrams_map[b] = dict()
                stem_bigrams_map[b][bigram_org] = stem_bigrams_map[b].get(bigram_org, 0)+1

    return word_list, bigrams

# keeps track of the exact form of the stemmed bigrams, not only the one of the words 

def stemmed(sentences):
    """
    reduce sentence terms to stemmed representations
    """
    stemmer = PorterStemmer()
    def stem(sentence):
        return ' '.join([stemmer.stem(word) for word in sentence.split()])
    return [stem(sentence) for sentence in sentences] 

def load_data(dname):
    stemmer = PorterStemmer()
    qids, questions, answers, labels = [], [], [], []
    print('Load folder ' + dname)
    with open(dname+'a.toks', encoding='utf-8') as f:
        for line in f:
            question = line.strip().split()
            question = [stemmer.stem(word) for word in question]
            questions.append(question)
    with open(dname+'b.toks', encoding='utf-8') as f:
        for line in f:
            answer = line.strip().split()
            answer_list = []
            for word in answer:
                try:
                    answer_list.append(stemmer.stem(word))
                except Exception as e:
                    print("couldn't stem the word:" + word)
            answers.append(answer_list)
    with open(dname + 'id.txt', encoding='utf-8') as f:
        for line in f:
            qids.append(line.strip())
    with open(dname + 'sim.txt', encoding='utf-8') as f:
        for line in f:
            labels.append(int(line.strip()))
    return qids, questions, answers, labels 

def read_in_data(datapath, set_name, file, stop_and_stem=False, stop_punct=False, dash_split=False):
    data = []
    with open(os.path.join(datapath, set_name, file)) as inf:
        data = [line.strip() for line in inf.readlines()]

        if dash_split:
            def split_hyphenated_words(sentence):
                rtokens = []
                for term in sentence.split():
                    for t in term.split('-'):
                        if t:
                            rtokens.append(t)
                return ' '.join(rtokens)
            data = [split_hyphenated_words(sentence) for sentence in data]

        if stop_punct:
            regex = re.compile('[{}]'.format(re.escape(string.punctuation)))
            def remove_punctuation(sentence):
                rtokens = []
                for term in sentence.split():
                    for t in regex.sub(' ', term).strip().split():
                        if t:
                            rtokens.append(t)
                return ' '.join(rtokens)
            data = [remove_punctuation(sentence) for sentence in data]

        if stop_and_stem:
            stemmer = PorterStemmer()
            stoplist = set(stopwords.words('english'))
            def stop_stem(sentence):
                return ' '.join([stemmer.stem(word) for word in sentence.split() \
                                                        if word not in stoplist])
            data = [stop_stem(sentence) for sentence in data]
    return data 

def sim_ngram_sentence(self, question, sentence,nGram):
        #considering stop words as well
        ps = PorterStemmer()
        getToken = lambda question:[ ps.stem(w.lower()) for w in word_tokenize(question) ]
        getNGram = lambda tokens,n:[ " ".join([tokens[index+i] for i in range(0,n)]) for index in range(0,len(tokens)-n+1)]
        qToken = getToken(question)
        sToken = getToken(sentence)

        if(len(qToken) > nGram):
            q3gram = set(getNGram(qToken,nGram))
            s3gram = set(getNGram(sToken,nGram))
            if(len(s3gram) < nGram):
                return 0
            qLen = len(q3gram)
            sLen = len(s3gram)
            sim = len(q3gram.intersection(s3gram)) / len(q3gram.union(s3gram))
            return sim
        else:
            return 0
    
    # Compute similarity between sentence and queryVector based on number of 
    # common words in both sentence. It doesn't consider occurance of words
    # Input:
    #       queryVector(dict)   : Dictionary of words in question
    #       sentence(str)       : Sentence string
    # Ouput:
    #       sim(float)          : Similarity Coefficient 

def __init__(self, question, useStemmer = False, useSynonyms = False, removeStopwords = False):
        self.question = question
        self.useStemmer = useStemmer
        self.useSynonyms = useSynonyms
        self.removeStopwords = removeStopwords
        self.stopWords = stopwords.words("english")
        self.stem = lambda k : k.lower()
        if self.useStemmer:
            ps = PorterStemmer()
            self.stem = ps.stem
        self.qType = self.determineQuestionType(question)
        self.searchQuery = self.buildSearchQuery(question)
        self.qVector = self.getQueryVector(self.searchQuery)
        self.aType = self.determineAnswerType(question)
    
    # To determine type of question by analyzing POS tag of question from Penn 
    # Treebank tagset
    #
    # Input:
    #           question(str) : Question string
    # Output:
    #           qType(str) : Type of question among following
    #                   [ WP ->  who
    #                     WDT -> what, why, how
    #                     WP$ -> whose
    #                     WRB -> where ] 

def __init__(self, rouge_types, use_stemmer=False):
    """Initializes a new RougeScorer.
    Valid rouge types that can be computed are:
      rougen (e.g. rouge1, rouge2): n-gram based scoring.
      rougeL: Longest common subsequence based scoring.
    Args:
      rouge_types: A list of rouge types to calculate.
      use_stemmer: Bool indicating whether Porter stemmer should be used to
        strip word suffixes to improve matching.
    Returns:
      A dict mapping rouge types to Score tuples.
    """

    self.rouge_types = rouge_types
    self._stemmer = porter.PorterStemmer() if use_stemmer else None 

def _enum_stem_match(enum_hypothesis_list, enum_reference_list, stemmer = PorterStemmer()):
    """
    Stems each word and matches them in hypothesis and reference 
    and returns a word mapping between enum_hypothesis_list and 
    enum_reference_list based on the enumerated word id. The function also 
    returns a enumerated list of unmatched words for hypothesis and reference.

    :param enum_hypothesis_list:
    :type enum_hypothesis_list:
    :param enum_reference_list:
    :type enum_reference_list:
    :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer())
    :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method
    :return: enumerated matched tuples, enumerated unmatched hypothesis tuples, 
             enumerated unmatched reference tuples
    :rtype: list of 2D tuples, list of 2D tuples,  list of 2D tuples
    """
    stemmed_enum_list1 = [(word_pair[0],stemmer.stem(word_pair[1])) \
                          for word_pair in enum_hypothesis_list]

    stemmed_enum_list2 = [(word_pair[0],stemmer.stem(word_pair[1])) \
                          for word_pair in enum_reference_list]

    word_match, enum_unmat_hypo_list, enum_unmat_ref_list = \
                    _match_enums(stemmed_enum_list1, stemmed_enum_list2)

    enum_unmat_hypo_list = list(zip(*enum_unmat_hypo_list)) if len(enum_unmat_hypo_list)>0 else []

    enum_unmat_ref_list = list(zip(*enum_unmat_ref_list)) if len(enum_unmat_ref_list)>0 else []

    enum_hypothesis_list = list(filter(lambda x:x[0] not in enum_unmat_hypo_list,  
                                       enum_hypothesis_list))

    enum_reference_list = list(filter(lambda x:x[0] not in enum_unmat_ref_list, 
                                      enum_reference_list))

    return word_match, enum_hypothesis_list, enum_reference_list 

def _enum_allign_words(enum_hypothesis_list, enum_reference_list, 
                       stemmer=PorterStemmer(), wordnet = wordnet):
    """
    Aligns/matches words in the hypothesis to reference by sequentially 
    applying exact match, stemmed match and wordnet based synonym match. 
    in case there are multiple matches the match which has the least number
    of crossing is chosen. Takes enumerated list as input instead of 
    string input

    :param enum_hypothesis_list: enumerated hypothesis list
    :param enum_reference_list: enumerated reference list
    :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer())
    :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method
    :param wordnet: a wordnet corpus reader object (default nltk.corpus.wordnet)
    :type wordnet: WordNetCorpusReader
    :return: sorted list of matched tuples, unmatched hypothesis list, 
             unmatched reference list
    :rtype: list of tuples, list of tuples, list of tuples
    """
    exact_matches, enum_hypothesis_list, enum_reference_list = \
        _match_enums(enum_hypothesis_list, enum_reference_list)

    stem_matches, enum_hypothesis_list, enum_reference_list = \
        _enum_stem_match(enum_hypothesis_list, enum_reference_list,
                         stemmer = stemmer)

    wns_matches, enum_hypothesis_list, enum_reference_list = \
        _enum_wordnetsyn_match(enum_hypothesis_list, enum_reference_list, 
                               wordnet = wordnet)

    return (sorted(exact_matches + stem_matches + wns_matches, key=lambda wordpair:wordpair[0]), 
            enum_hypothesis_list, enum_reference_list) 

def __init__(self, ignore_stopwords=False):
        _LanguageSpecificStemmer.__init__(self, ignore_stopwords)
        porter.PorterStemmer.__init__(self) 

def __init__(self, model: Model, dataset_reader: DatasetReader) -> None:
        super().__init__(model, dataset_reader)
        self._stemmer = PorterStemmer()
        self._stop_words = set(stopwords.words('english')) 

def save_lexicon(output, scored_terms, term_freq, stem, score):
    ps = PorterStemmer()
    f1 = open(output, "w")
    f2 = open(output[0:len(output)-len(output.split(".")[len(output.split("."))-1])-1]+"_with_scores_%s.txt"%score,"w")
    print "Saving the lexicon to file..."
    for i,t in enumerate(scored_terms):
        print>>f1,t[0]
        print>>f2,"%s,%s,%s"%(t[0],t[1],term_freq[stem[i]])
    print "The Lexicon is ready!" 

def __init__(self, ignore_stopwords=False):
        _LanguageSpecificStemmer.__init__(self, ignore_stopwords)
        porter.PorterStemmer.__init__(self) 

def __init__(self, lang="spanish"):
        """
        Initializes the parameters for specific language
        """
        self.languages = ["spanish", "english", "italian", "german", "arabic"]
        self.lang = lang

        if self.lang not in SnowballStemmer.languages:
            raise LangDependencyError("Language not supported for stemming: " + lang)
        if self.lang == "english":
            self.stemmer = PorterStemmer()
        else:
            self.stemmer = SnowballStemmer(self.lang) 

def text_preprocess(self, text, stemming=True, lemmatization=False, join=False):

        for func in self.cleanup_functions:
            text = func(text)

        text = re.sub("[^a-zA-Z0-9]", " ", text)

        if lemmatization:
            text = [word.lemma_ for word in get_nlp()(text)]
        elif stemming:
            ps = PorterStemmer()
            tokenized_text = (
                word_tokenize(text.lower())
                if self.nltk_tokenizer
                else text.lower().split()
            )
            text = [
                ps.stem(word)
                for word in tokenized_text
                if word not in set(stopwords.words("english")) and len(word) > 1
            ]
        else:
            text = text.split()

        if join:
            return " ".join(word for word in text)
        return text 

def __init__(self, ignore_stopwords=False):
        _LanguageSpecificStemmer.__init__(self, ignore_stopwords)
        porter.PorterStemmer.__init__(self) 

def __init__(self, full_word):
        self.full_word = full_word
        # TODO: Lemmatization requires downloads
        # wnl = WordNetLemmatizer()
        # lemmas = [wnl.lemmatize(token) for token in tokens]
        self.stem = PorterStemmer().stem(full_word).lower() 

def tiny_tokenize_xml(text, stem=False, stop_words=[]):
    return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
                        re.sub('[%s]' % re.escape(string.punctuation), ' ', text.encode(encoding='ascii', errors='ignore'))) if
                        not token.isdigit() and not token in stop_words] 

def test_do_not_use_stemmer_when_UnicodeDecodeError():
    """Test DictionaryMatch when stemmer causes UnicodeDecodeError."""
    stemmer = PorterStemmer()
    matcher = DictionaryMatch(d=["is"], stemmer=stemmer)
    # _stem(w) should return a word stem.
    assert matcher._stem("caresses") == "caress"

    stemmer.stem = Mock(
        side_effect=UnicodeDecodeError("dummycodec", b"\x00\x00", 1, 2, "Dummy  !")
    )
    matcher = DictionaryMatch(d=["is"], stemmer=stemmer)
    # _stem(w) should return w as stemmer.stem raises UnicodeDecodeError.
    assert matcher._stem("caresses") == "caresses" 

def get_stemmed_separate(indeed_reviews_db, glassdoor_reviews_db):
    separate = get_separate_reviews(indeed_reviews_db, glassdoor_reviews_db)
    stemmer = PorterStemmer()
    stemmed_reviews = []
    for review in separate:
        stemmed_reviews.append(' '.join([stemmer.stem(word) for sent in sent_tokenize(review) for word in word_tokenize(sent.lower())]))
    return stemmed_reviews 

def get_stemmed_combined_reviews(indeed_reviews_db, glassdoor_reviews_db):
    combined = get_combined_reviews(indeed_reviews_db, glassdoor_reviews_db)

    stemmer = PorterStemmer()
    stemmed_reviews = []
    for review in combined:
        stemmed_reviews.append(' '.join([stemmer.stem(word) for sent in sent_tokenize(review) for word in word_tokenize(sent.lower())]))

    return stemmed_reviews 

def addtoVocab(self, words):
        #stemmer = PorterStemmer()
        w_list = self.removeStopWords(words)
        for word in w_list:
            self.vocabulary[word] += 1
        return w_list