<jupyter_start><jupyter_text>Stemming words<jupyter_code># PorterStemmer

from nltk.stem import PorterStemmer

porter = PorterStemmer()

porterWords = [porter.stem(w) for w in nltk_word]

print(porterWords)
from nltk.stem import PorterStemmer
porter_stemmer = PorterStemmer()

words = ["multidimensional", "coming", "sing", "stemming" ]

[porter_stemmer.stem(w) for w in words]

#   Porter Stemming Algorithm   - http://snowball.tartarus.org/algorithms/porter/stemmer.html
# SnowballStemmer

from nltk.stem import SnowballStemmer

snowball = SnowballStemmer('english')

snowballWords = [snowball.stem(w) for w in nltk_word]

print(snowballWords)
# LancasterStemmer

from nltk.stem import LancasterStemmer

lancaster = LancasterStemmer()

lancasterWords = [lancaster.stem(w) for w in nltk_word]

print(lancasterWords)
# RegexpStemmer

from nltk.stem import RegexpStemmer

regexp = RegexpStemmer('ing')

regexpWords = [regexp.stem(w) for w in nltk_word]

print(regexpWords)<jupyter_output>['When', 'I', 'first', 'met', 'him', 'he', 'was', "n't", 'very', 'quiet', '.', 'He', 'spoke', 'without', 'stopp', 'dur', 'the', 'entire', 'two', 'hour', 'long', 'journey', 'from', 'Washton', 'to', 'New York', '.']<jupyter_text>Lemmatizing words<jupyter_code># lemmatize as noun (default)   :   Valid options are `"n"` for nouns, `"v"` for verbs, `"a"` for adjectives, `"r"` for adverbs and `"s"` for satellite adjectives.

#   import nltk
#   nltk.download('wordnet')
#   nltk.download('omw-1.4')

from nltk.stem import WordNetLemmatizer

lemma = WordNetLemmatizer()

lemmaWords = [lemma.lemmatize(w) for w in nltk_word]

print(lemmaWords)
# lemmatize as verbs

lemmaWords = [lemma.lemmatize(w, pos = 'v') for w in nltk_word]

print(lemmaWords)
# lemmatize as adjectives

lemma.lemmatize('better', pos = 'a')
# lemmatize as adverbs

lemma.lemmatize('better', pos = 'r')
import spacy
#spaCy determines the part-of-speech tag by default and assigns the corresponding lemma. It comes with a bunch of prebuilt models where the 'en' we just downloaded above is one of the standard ones for english.

sentence = ' '.join(nltk_word)

nlp = spacy.load('en_core_web_sm')
doc = nlp(sentence)

# Extract the lemma for each token and join
print([token.lemma_ for token in doc])
doc = nlp("I'm coming in a multimensional")

[token.lemma_ for token in doc]

doc = nlp(" ".join(filteredWords))

print([token.lemma_ for token in doc])<jupyter_output>['when', 'I', 'first', 'meet', 'not', 'quiet', 'he', 'speak', 'without', 'stop', 'entire', 'two', 'hour', 'long', 'journey', 'Washington', 'New', 'York']
{'fraction_non_alphanumeric': 0.14840714840714841, 'fraction_numerical': 0.000777000777000777, 'mean_word_length': 5.63659793814433, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '113826', 'license': 'MIT', 'n_tokens_mistral': 1035, 'n_tokens_neox': 965, 'n_words': 262, 'path': '2021-22/Code_03_NLP_Basics/03_c_Normalization_NLP_Basics.ipynb', 'repo_name': 'ranieri-unimi/notes-ferrara-2022', 'size': '2574'}