<jupyter_start><jupyter_text>StopwordsStopwords are non-informative words that we want to take out of the text before performing analysis on it. Luckily, NLTK has a ready-made list of such words that we can use to preprocess text...but is this enough?<jupyter_code>from nltk.corpus import reuters, stopwords
from nltk.tokenize import word_tokenize, sent_tokenize
article = reuters.raw(fileids=reuters.fileids(categories='crude')[0])
print(article)
sentence = sent_tokenize(article)[1]
print(sentence)
words = word_tokenize(sentence)
print(words)<jupyter_output>['MITI', 'is', 'expected', 'to', 'lower', 'the', 'projection', 'for', 'primary', 'energy', 'supplies', 'in', 'the', 'year', '2000', 'to', '550', 'mln', 'kilolitres', '(', 'kl', ')', 'from', '600', 'mln', ',', 'they', 'said', '.']<jupyter_text>NLTK Stopwords<jupyter_code># NLTK list of stopwords
sw = set(stopwords.words('english'))
sw
# Instantiate the list of stopwords, then convert all to lowercase
first_result = [word.lower() for word in words if word.lower() not in sw]
# Print result
print(f'{first_result}')
# We can define our own list of stopwords to add to the default nltk words
sw_addons = {'said', 'sent', 'found', 'including', 'today', 'announced', 'week', 'basically', 'also'}
# Print result
output = [word.lower() for word in words if word.lower() not in sw.union(sw_addons)]
print(f'{output}')<jupyter_output>['miti', 'expected', 'lower', 'projection', 'primary', 'energy', 'supplies', 'year', '2000', '550', 'mln', 'kilolitres', '(', 'kl', ')', '600', 'mln', ',', '.']<jupyter_text>Getting Rid of Non-Alpha Characters: Regex<jupyter_code># Import regular expressions library
import re
# Substitute everything that is NOT a letter with empty string
import re
regex = re.compile("[^a-zA-Z ]")
# Tokenize re_clean, convert to lower case, and remove stop words 

re_clean = regex.sub('', article)
#print(f'{re_clean}')
sentence = sent_tokenize(re_clean)
#print(f'{sentence}')
words = word_tokenize(sentence[0])
output = [word.lower() for word in words if word.lower() not in sw.union(sw_addons)]
# Print result
print(f'{output}')<jupyter_output>['japan', 'revise', 'longterm', 'energy', 'demand', 'downwards', 'ministry', 'international', 'trade', 'industry', 'miti', 'revise', 'longterm', 'energy', 'supplydemand', 'outlook', 'august', 'meet', 'forecast', 'downtrend', 'japanese', 'energy', 'demand', 'ministry', 'officials', 'miti', 'expected', 'lower', 'projection', 'primary', 'energy', 'supplies', 'year', 'mln', 'kilolitres', 'kl', 'mln', 'decision', 'follows', 'emergence', 'structural', 'changes', 'japanese', 'industry', 'following', 'rise', 'value', 'yen', 'decline', 'domestic', 'electric', 'power', 'demand', 'miti', 'planning', 'work', 'revised', 'energy', 'supplydemand', 'outlook', 'deliberations', 'committee', 'meetings', 'agency', 'natural', 'resources', 'energy', 'officials', 'miti', 'review', 'breakdown', 'energy', 'supply', 'sources', 'oil', 'nuclear', 'coal', 'natural', 'gas', 'nuclear', 'energy', 'provided', 'bulk', 'japans', 'electric', 'power', 'fiscal', 'year', 'ended', 'march', 'supplying', 'estimated', 'pct', 'k[...]
{'fraction_non_alphanumeric': 0.20498880716341542, 'fraction_numerical': 0.007355292612727854, 'mean_word_length': 7.363636363636363, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '231006', 'license': 'MIT', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1059, 'n_words': 317, 'path': 'PythonJupyterNotebooks/Week12-Day1-Activity3-stopwords.ipynb', 'repo_name': 'nomadic-me/CodingCheatSheets', 'size': '3127'}