<jupyter_start><jupyter_text>Word Tokenisation============<jupyter_code>text = "The quick brown fox jumped over the lazy dog"

tokens = text.split()
print(tokens)
text = "The quick brown fox, and an Oxford comma"

tokens = text.split()
print(tokens)
from nltk.tokenize import word_tokenize

text = "The quick brown fox, and an Oxford comma"

tokens = word_tokenize(text)
print(tokens)
from nltk.tokenize import word_tokenize

text = "Tweet about #NLProc @PyConUK :)"

tokens = word_tokenize(text)
print(tokens)
from nltk.tokenize import TweetTokenizer

text = "Tweet about #NLProc @PyConUK :)"

tokenizer = TweetTokenizer()
tokens = tokenizer.tokenize(text)
print(tokens)
from nltk.tokenize import TweetTokenizer
text = "Tweet about #NLProc @PyConUK :)"

tokenizer = TweetTokenizer(strip_handles=True)
tokens = tokenizer.tokenize(text)
print(tokens)
from nltk.tokenize import word_tokenize
text = "How about currencies (like £100,000.00) and dates (like 19th September)"

tokens = word_tokenize(text)
print(tokens)<jupyter_output><empty_output><jupyter_text>Stemming=====<jupyter_code>from nltk.stem import PorterStemmer

s = PorterStemmer()
print(s.stem('Having'))
print(s.stem('Have'))
print(s.stem('Had'))

print(s.stem('Fishing'))
print(s.stem('Fish'))
print(s.stem('Fisher'))
print(s.stem('Fishes'))
print(s.stem('Fished'))<jupyter_output><empty_output><jupyter_text>Lemmatisation=====Lemmatisation is similar to stemming, as it produces a normalised version of the input word.The output is a lemma, i.e. a proper word (different from stemming)The input word is lemmatised according to its Part-of-Speech (POS) tag, i.e. verb, noun, etc.<jupyter_code># You'll need the "wordnet" package from NLTK data
# python -m nltk.downloader wordnet
from nltk.stem import WordNetLemmatizer

s = WordNetLemmatizer()
print(s.lemmatize('having', pos='v'))
print(s.lemmatize('have', pos='v'))
print(s.lemmatize('had', pos='v'))

print(s.lemmatize('fishing', pos='v'))
print(s.lemmatize('fish', pos='v'))
print(s.lemmatize('fisher', pos='n'))
print(s.lemmatize('fishes', pos='v'))
print(s.lemmatize('fished', pos='v'))

print(s.lemmatize('am', pos='v'))
print(s.lemmatize('is', pos='v'))
print(s.lemmatize('was', pos='v'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15625, 'fraction_numerical': 0.004464285714285714, 'mean_word_length': 7.3, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '476748', 'license': 'MIT', 'n_tokens_mistral': 846, 'n_tokens_neox': 801, 'n_words': 192, 'path': 'notebooks/more_text_preprocessing_examples.ipynb', 'repo_name': 'MeetRishabh03/NLP-NLTK', 'size': '2240'}