Python nltk.stem.porter.PorterStemmer() Examples
The following are 30
code examples of nltk.stem.porter.PorterStemmer().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
nltk.stem.porter
, or try the search function
.
Example #1
Source File: meteor_score.py From V1EngineeringInc-Docs with Creative Commons Attribution Share Alike 4.0 International | 6 votes |
def allign_words(hypothesis, reference, stemmer = PorterStemmer(), wordnet = wordnet): """ Aligns/matches words in the hypothesis to reference by sequentially applying exact match, stemmed match and wordnet based synonym match. In case there are multiple matches the match which has the least number of crossing is chosen. :param hypothesis: hypothesis string :param reference: reference string :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer()) :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method :param wordnet: a wordnet corpus reader object (default nltk.corpus.wordnet) :type wordnet: WordNetCorpusReader :return: sorted list of matched tuples, unmatched hypothesis list, unmatched reference list :rtype: list of tuples, list of tuples, list of tuples """ enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference) return _enum_allign_words(enum_hypothesis_list, enum_reference_list, stemmer= stemmer, wordnet= wordnet)
Example #2
Source File: evaluate.py From sota-extractor with Apache License 2.0 | 6 votes |
def load(tdb): # load the tasks and arxiv metadata stemmer = PorterStemmer() tdb.load_tasks("data/tasks/nlpprogress.json") tdb.load_synonyms(["data/tasks/synonyms.csv"]) arxiv = serialization.load( "data/arxiv_aclweb.json.gz", fmt=serialization.Format.json_gz ) for a in arxiv: if a["abstract"] is None: a["abstract"] = "" # require and normalise arxiv titles arxiv = [a for a in arxiv if "title" in a and a["title"] is not None] for a in arxiv: a["title"] = re.sub(" +", " ", a["title"].replace("\n", " ")) a["title_lower"] = a["title"].lower() a["abstract_lower"] = a["abstract"].lower() a["title_stem"] = stemmer.stem(a["title"]) a["abstract_stem"] = stemmer.stem(a["abstract"]) return arxiv
Example #3
Source File: meteor_score.py From V1EngineeringInc-Docs with Creative Commons Attribution Share Alike 4.0 International | 6 votes |
def stem_match(hypothesis, reference, stemmer = PorterStemmer()): """ Stems each word and matches them in hypothesis and reference and returns a word mapping between hypothesis and reference :param hypothesis: :type hypothesis: :param reference: :type reference: :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer()) :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method :return: enumerated matched tuples, enumerated unmatched hypothesis tuples, enumerated unmatched reference tuples :rtype: list of 2D tuples, list of 2D tuples, list of 2D tuples """ enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference) return _enum_stem_match(enum_hypothesis_list, enum_reference_list, stemmer = stemmer)
Example #4
Source File: DocumentRetrievalModel.py From Factoid-based-Question-Answer-Chatbot with MIT License | 6 votes |
def __init__(self,paragraphs,removeStopWord = False,useStemmer = False): self.idf = {} # dict to store IDF for words in paragraph self.paragraphInfo = {} # structure to store paragraphVector self.paragraphs = paragraphs self.totalParas = len(paragraphs) self.stopwords = stopwords.words('english') self.removeStopWord = removeStopWord self.useStemmer = useStemmer self.vData = None self.stem = lambda k:k.lower() if(useStemmer): ps = PorterStemmer() self.stem = ps.stem # Initialize self.computeTFIDF() # Return term frequency for Paragraph # Input: # paragraph(str): Paragraph as a whole in string format # Output: # wordFrequence(dict) : Dictionary of word and term frequency
Example #5
Source File: test_matchers.py From fonduer with MIT License | 6 votes |
def test_dictionary_match(doc_setup): """Test DictionaryMatch matcher.""" doc = doc_setup space = MentionNgrams(n_min=1, n_max=1) # Test with a list of str matcher = DictionaryMatch(d=["this"]) assert set(tc.get_span() for tc in matcher.apply(space.apply(doc))) == {"This"} # Test without a dictionary with pytest.raises(Exception): DictionaryMatch() # TODO: test with plural words matcher = DictionaryMatch(d=["is"], stemmer=PorterStemmer()) assert set(tc.get_span() for tc in matcher.apply(space.apply(doc))) == {"is"} # Test if matcher raises an error when _f is given non-TemporarySpanMention matcher = DictionaryMatch(d=["this"]) with pytest.raises(ValueError): list(matcher.apply(doc.sentences[0].words))
Example #6
Source File: preprocessing.py From KATE with BSD 3-Clause "New" or "Revised" License | 6 votes |
def tiny_tokenize(text, stem=False, stop_words=[]): words = [] for token in wordpunct_tokenize(re.sub('[%s]' % re.escape(string.punctuation), ' ', \ text.decode(encoding='UTF-8', errors='ignore'))): if not token.isdigit() and not token in stop_words: if stem: try: w = EnglishStemmer().stem(token) except Exception as e: w = token else: w = token words.append(w) return words # return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize( # re.sub('[%s]' % re.escape(string.punctuation), ' ', text.decode(encoding='UTF-8', errors='ignore'))) if # not token.isdigit() and not token in stop_words]
Example #7
Source File: DocumentRetrievalModel.py From Factoid-based-Question-Answer-Chatbot with MIT License | 6 votes |
def sim_sentence(self, queryVector, sentence): sentToken = word_tokenize(sentence) ps = PorterStemmer() for index in range(0,len(sentToken)): sentToken[index] = ps.stem(sentToken[index]) sim = 0 for word in queryVector.keys(): w = ps.stem(word) if w in sentToken: sim += 1 return sim/(len(sentToken)*len(queryVector.keys())) # Get Named Entity from the sentence in form of PERSON, GPE, & ORGANIZATION # Input: # answers(list) : List of potential sentence containing answer # Output: # chunks(list) : List of tuple with entity and name in ranked # order
Example #8
Source File: overlap_features.py From castor with Apache License 2.0 | 5 votes |
def load_data(dname): stemmer = PorterStemmer() qids, questions, answers, labels = [], [], [], [] print('Load folder ' + dname) with open(dname+'a.toks', encoding='utf-8') as f: for line in f: question = line.strip().split() question = [stemmer.stem(word) for word in question] questions.append(question) with open(dname+'b.toks', encoding='utf-8') as f: for line in f: answer = line.strip().split() answer_list = [] for word in answer: try: answer_list.append(stemmer.stem(word)) except Exception as e: print("couldn't stem the word:" + word) answers.append(answer_list) with open(dname+'id.txt', encoding='utf-8') as f: for line in f: qids.append(line.strip()) with open(dname+'sim.txt', encoding='utf-8') as f: for line in f: labels.append(int(line.strip())) return qids, questions, answers, labels
Example #9
Source File: read.py From CrisisLex with MIT License | 5 votes |
def get_stemmed_terms_list(doc, stem_words_map = None, stem_bigrams_map = None): ps = PorterStemmer() local_map = dict() word_list = [] clean_doc = [(w.strip()).lower() for w in doc.split() if len(w) in range(3,16)] filtered_words = [w.strip('.,;?!:)(#') for w in clean_doc if not w.strip('.,;?!:)(#') in stopwords.words('english')] for w in filtered_words: if w.isalpha(): w_temp = ps.stem_word(w) if stem_words_map is not None: if w_temp not in stem_words_map: stem_words_map[w_temp] = dict() stem_words_map[w_temp][w] = stem_words_map[w_temp].get(w, 0)+1 local_map[w_temp] = w word_list.append(w_temp) bigrams = nltk.bigrams(word_list) for b in bigrams: bigram_org = (local_map[b[0]],local_map[b[1]]) if stem_bigrams_map is not None: if b not in stem_bigrams_map: stem_bigrams_map[b] = dict() stem_bigrams_map[b][bigram_org] = stem_bigrams_map[b].get(bigram_org, 0)+1 return word_list, bigrams # keeps track of the exact form of the stemmed bigrams, not only the one of the words
Example #10
Source File: external_features.py From castor with Apache License 2.0 | 5 votes |
def stemmed(sentences): """ reduce sentence terms to stemmed representations """ stemmer = PorterStemmer() def stem(sentence): return ' '.join([stemmer.stem(word) for word in sentence.split()]) return [stem(sentence) for sentence in sentences]
Example #11
Source File: overlap_features.py From castor with Apache License 2.0 | 5 votes |
def load_data(dname): stemmer = PorterStemmer() qids, questions, answers, labels = [], [], [], [] print('Load folder ' + dname) with open(dname+'a.toks', encoding='utf-8') as f: for line in f: question = line.strip().split() question = [stemmer.stem(word) for word in question] questions.append(question) with open(dname+'b.toks', encoding='utf-8') as f: for line in f: answer = line.strip().split() answer_list = [] for word in answer: try: answer_list.append(stemmer.stem(word)) except Exception as e: print("couldn't stem the word:" + word) answers.append(answer_list) with open(dname + 'id.txt', encoding='utf-8') as f: for line in f: qids.append(line.strip()) with open(dname + 'sim.txt', encoding='utf-8') as f: for line in f: labels.append(int(line.strip())) return qids, questions, answers, labels
Example #12
Source File: qa-data-only-idf.py From castor with Apache License 2.0 | 5 votes |
def read_in_data(datapath, set_name, file, stop_and_stem=False, stop_punct=False, dash_split=False): data = [] with open(os.path.join(datapath, set_name, file)) as inf: data = [line.strip() for line in inf.readlines()] if dash_split: def split_hyphenated_words(sentence): rtokens = [] for term in sentence.split(): for t in term.split('-'): if t: rtokens.append(t) return ' '.join(rtokens) data = [split_hyphenated_words(sentence) for sentence in data] if stop_punct: regex = re.compile('[{}]'.format(re.escape(string.punctuation))) def remove_punctuation(sentence): rtokens = [] for term in sentence.split(): for t in regex.sub(' ', term).strip().split(): if t: rtokens.append(t) return ' '.join(rtokens) data = [remove_punctuation(sentence) for sentence in data] if stop_and_stem: stemmer = PorterStemmer() stoplist = set(stopwords.words('english')) def stop_stem(sentence): return ' '.join([stemmer.stem(word) for word in sentence.split() \ if word not in stoplist]) data = [stop_stem(sentence) for sentence in data] return data
Example #13
Source File: DocumentRetrievalModel.py From Factoid-based-Question-Answer-Chatbot with MIT License | 5 votes |
def sim_ngram_sentence(self, question, sentence,nGram): #considering stop words as well ps = PorterStemmer() getToken = lambda question:[ ps.stem(w.lower()) for w in word_tokenize(question) ] getNGram = lambda tokens,n:[ " ".join([tokens[index+i] for i in range(0,n)]) for index in range(0,len(tokens)-n+1)] qToken = getToken(question) sToken = getToken(sentence) if(len(qToken) > nGram): q3gram = set(getNGram(qToken,nGram)) s3gram = set(getNGram(sToken,nGram)) if(len(s3gram) < nGram): return 0 qLen = len(q3gram) sLen = len(s3gram) sim = len(q3gram.intersection(s3gram)) / len(q3gram.union(s3gram)) return sim else: return 0 # Compute similarity between sentence and queryVector based on number of # common words in both sentence. It doesn't consider occurance of words # Input: # queryVector(dict) : Dictionary of words in question # sentence(str) : Sentence string # Ouput: # sim(float) : Similarity Coefficient
Example #14
Source File: ProcessedQuestion.py From Factoid-based-Question-Answer-Chatbot with MIT License | 5 votes |
def __init__(self, question, useStemmer = False, useSynonyms = False, removeStopwords = False): self.question = question self.useStemmer = useStemmer self.useSynonyms = useSynonyms self.removeStopwords = removeStopwords self.stopWords = stopwords.words("english") self.stem = lambda k : k.lower() if self.useStemmer: ps = PorterStemmer() self.stem = ps.stem self.qType = self.determineQuestionType(question) self.searchQuery = self.buildSearchQuery(question) self.qVector = self.getQueryVector(self.searchQuery) self.aType = self.determineAnswerType(question) # To determine type of question by analyzing POS tag of question from Penn # Treebank tagset # # Input: # question(str) : Question string # Output: # qType(str) : Type of question among following # [ WP -> who # WDT -> what, why, how # WP$ -> whose # WRB -> where ]
Example #15
Source File: rouge_scorer.py From compare-mt with BSD 3-Clause "New" or "Revised" License | 5 votes |
def __init__(self, rouge_types, use_stemmer=False): """Initializes a new RougeScorer. Valid rouge types that can be computed are: rougen (e.g. rouge1, rouge2): n-gram based scoring. rougeL: Longest common subsequence based scoring. Args: rouge_types: A list of rouge types to calculate. use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes to improve matching. Returns: A dict mapping rouge types to Score tuples. """ self.rouge_types = rouge_types self._stemmer = porter.PorterStemmer() if use_stemmer else None
Example #16
Source File: meteor_score.py From V1EngineeringInc-Docs with Creative Commons Attribution Share Alike 4.0 International | 5 votes |
def _enum_stem_match(enum_hypothesis_list, enum_reference_list, stemmer = PorterStemmer()): """ Stems each word and matches them in hypothesis and reference and returns a word mapping between enum_hypothesis_list and enum_reference_list based on the enumerated word id. The function also returns a enumerated list of unmatched words for hypothesis and reference. :param enum_hypothesis_list: :type enum_hypothesis_list: :param enum_reference_list: :type enum_reference_list: :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer()) :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method :return: enumerated matched tuples, enumerated unmatched hypothesis tuples, enumerated unmatched reference tuples :rtype: list of 2D tuples, list of 2D tuples, list of 2D tuples """ stemmed_enum_list1 = [(word_pair[0],stemmer.stem(word_pair[1])) \ for word_pair in enum_hypothesis_list] stemmed_enum_list2 = [(word_pair[0],stemmer.stem(word_pair[1])) \ for word_pair in enum_reference_list] word_match, enum_unmat_hypo_list, enum_unmat_ref_list = \ _match_enums(stemmed_enum_list1, stemmed_enum_list2) enum_unmat_hypo_list = list(zip(*enum_unmat_hypo_list)) if len(enum_unmat_hypo_list)>0 else [] enum_unmat_ref_list = list(zip(*enum_unmat_ref_list)) if len(enum_unmat_ref_list)>0 else [] enum_hypothesis_list = list(filter(lambda x:x[0] not in enum_unmat_hypo_list, enum_hypothesis_list)) enum_reference_list = list(filter(lambda x:x[0] not in enum_unmat_ref_list, enum_reference_list)) return word_match, enum_hypothesis_list, enum_reference_list
Example #17
Source File: meteor_score.py From V1EngineeringInc-Docs with Creative Commons Attribution Share Alike 4.0 International | 5 votes |
def _enum_allign_words(enum_hypothesis_list, enum_reference_list, stemmer=PorterStemmer(), wordnet = wordnet): """ Aligns/matches words in the hypothesis to reference by sequentially applying exact match, stemmed match and wordnet based synonym match. in case there are multiple matches the match which has the least number of crossing is chosen. Takes enumerated list as input instead of string input :param enum_hypothesis_list: enumerated hypothesis list :param enum_reference_list: enumerated reference list :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer()) :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method :param wordnet: a wordnet corpus reader object (default nltk.corpus.wordnet) :type wordnet: WordNetCorpusReader :return: sorted list of matched tuples, unmatched hypothesis list, unmatched reference list :rtype: list of tuples, list of tuples, list of tuples """ exact_matches, enum_hypothesis_list, enum_reference_list = \ _match_enums(enum_hypothesis_list, enum_reference_list) stem_matches, enum_hypothesis_list, enum_reference_list = \ _enum_stem_match(enum_hypothesis_list, enum_reference_list, stemmer = stemmer) wns_matches, enum_hypothesis_list, enum_reference_list = \ _enum_wordnetsyn_match(enum_hypothesis_list, enum_reference_list, wordnet = wordnet) return (sorted(exact_matches + stem_matches + wns_matches, key=lambda wordpair:wordpair[0]), enum_hypothesis_list, enum_reference_list)
Example #18
Source File: snowball.py From V1EngineeringInc-Docs with Creative Commons Attribution Share Alike 4.0 International | 5 votes |
def __init__(self, ignore_stopwords=False): _LanguageSpecificStemmer.__init__(self, ignore_stopwords) porter.PorterStemmer.__init__(self)
Example #19
Source File: bidaf_qa_predictor.py From ARC-Solvers with Apache License 2.0 | 5 votes |
def __init__(self, model: Model, dataset_reader: DatasetReader) -> None: super().__init__(model, dataset_reader) self._stemmer = PorterStemmer() self._stop_words = set(stopwords.words('english'))
Example #20
Source File: build.py From CrisisLex with MIT License | 5 votes |
def save_lexicon(output, scored_terms, term_freq, stem, score): ps = PorterStemmer() f1 = open(output, "w") f2 = open(output[0:len(output)-len(output.split(".")[len(output.split("."))-1])-1]+"_with_scores_%s.txt"%score,"w") print "Saving the lexicon to file..." for i,t in enumerate(scored_terms): print>>f1,t[0] print>>f2,"%s,%s,%s"%(t[0],t[1],term_freq[stem[i]]) print "The Lexicon is ready!"
Example #21
Source File: snowball.py From razzy-spinner with GNU General Public License v3.0 | 5 votes |
def __init__(self, ignore_stopwords=False): _LanguageSpecificStemmer.__init__(self, ignore_stopwords) porter.PorterStemmer.__init__(self)
Example #22
Source File: lang_dependency.py From b4msa with Apache License 2.0 | 5 votes |
def __init__(self, lang="spanish"): """ Initializes the parameters for specific language """ self.languages = ["spanish", "english", "italian", "german", "arabic"] self.lang = lang if self.lang not in SnowballStemmer.languages: raise LangDependencyError("Language not supported for stemming: " + lang) if self.lang == "english": self.stemmer = PorterStemmer() else: self.stemmer = SnowballStemmer(self.lang)
Example #23
Source File: similarity.py From bugbug with Mozilla Public License 2.0 | 5 votes |
def text_preprocess(self, text, stemming=True, lemmatization=False, join=False): for func in self.cleanup_functions: text = func(text) text = re.sub("[^a-zA-Z0-9]", " ", text) if lemmatization: text = [word.lemma_ for word in get_nlp()(text)] elif stemming: ps = PorterStemmer() tokenized_text = ( word_tokenize(text.lower()) if self.nltk_tokenizer else text.lower().split() ) text = [ ps.stem(word) for word in tokenized_text if word not in set(stopwords.words("english")) and len(word) > 1 ] else: text = text.split() if join: return " ".join(word for word in text) return text
Example #24
Source File: snowball.py From luscan-devel with GNU General Public License v2.0 | 5 votes |
def __init__(self, ignore_stopwords=False): _LanguageSpecificStemmer.__init__(self, ignore_stopwords) porter.PorterStemmer.__init__(self)
Example #25
Source File: lang_proc.py From SearchingReddit with MIT License | 5 votes |
def __init__(self, full_word): self.full_word = full_word # TODO: Lemmatization requires downloads # wnl = WordNetLemmatizer() # lemmas = [wnl.lemmatize(token) for token in tokens] self.stem = PorterStemmer().stem(full_word).lower()
Example #26
Source File: preprocessing.py From KATE with BSD 3-Clause "New" or "Revised" License | 5 votes |
def tiny_tokenize_xml(text, stem=False, stop_words=[]): return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize( re.sub('[%s]' % re.escape(string.punctuation), ' ', text.encode(encoding='ascii', errors='ignore'))) if not token.isdigit() and not token in stop_words]
Example #27
Source File: test_matchers.py From fonduer with MIT License | 5 votes |
def test_do_not_use_stemmer_when_UnicodeDecodeError(): """Test DictionaryMatch when stemmer causes UnicodeDecodeError.""" stemmer = PorterStemmer() matcher = DictionaryMatch(d=["is"], stemmer=stemmer) # _stem(w) should return a word stem. assert matcher._stem("caresses") == "caress" stemmer.stem = Mock( side_effect=UnicodeDecodeError("dummycodec", b"\x00\x00", 1, 2, "Dummy !") ) matcher = DictionaryMatch(d=["is"], stemmer=stemmer) # _stem(w) should return w as stemmer.stem raises UnicodeDecodeError. assert matcher._stem("caresses") == "caresses"
Example #28
Source File: reviews_data.py From company-reviews with MIT License | 5 votes |
def get_stemmed_separate(indeed_reviews_db, glassdoor_reviews_db): separate = get_separate_reviews(indeed_reviews_db, glassdoor_reviews_db) stemmer = PorterStemmer() stemmed_reviews = [] for review in separate: stemmed_reviews.append(' '.join([stemmer.stem(word) for sent in sent_tokenize(review) for word in word_tokenize(sent.lower())])) return stemmed_reviews
Example #29
Source File: reviews_data.py From company-reviews with MIT License | 5 votes |
def get_stemmed_combined_reviews(indeed_reviews_db, glassdoor_reviews_db): combined = get_combined_reviews(indeed_reviews_db, glassdoor_reviews_db) stemmer = PorterStemmer() stemmed_reviews = [] for review in combined: stemmed_reviews.append(' '.join([stemmer.stem(word) for sent in sent_tokenize(review) for word in word_tokenize(sent.lower())])) return stemmed_reviews
Example #30
Source File: preprocessing.py From TBBTCorpus with Apache License 2.0 | 5 votes |
def addtoVocab(self, words): #stemmer = PorterStemmer() w_list = self.removeStopWords(words) for word in w_list: self.vocabulary[word] += 1 return w_list