<jupyter_start><jupyter_text>Best Practices for Preprocessing Natural Language Data In this notebook, we improve the quality of our Project Gutenberg word vectors by adopting best-practices for preprocessing natural language data.**N.B.:** Some, all or none of these preprocessing steps may be helpful to a given downstream application.   Load dependencies<jupyter_code># the initial block is copied from creating_word_vectors_with_word2vec.ipynb
import nltk
from nltk import word_tokenize, sent_tokenize
import gensim
from gensim.models.word2vec import Word2Vec
from sklearn.manifold import TSNE
import pandas as pd
from bokeh.io import output_notebook, output_file
from bokeh.plotting import show, figure
%matplotlib inline
nltk.download('punkt')
# new!
import string
from nltk.corpus import stopwords
from nltk.stem.porter import *
from gensim.models.phrases import Phraser, Phrases
from keras.preprocessing.text import one_hot
nltk.download('stopwords')<jupyter_output>[nltk_data] Downloading package stopwords to /home/jovyan/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Load data<jupyter_code>nltk.download('gutenberg')
from nltk.corpus import gutenberg
gberg_sents = gutenberg.sents()<jupyter_output><empty_output><jupyter_text>Iteratively preprocess a sentence  a tokenized sentence:<jupyter_code>gberg_sents[4]<jupyter_output><empty_output><jupyter_text>to lowercase:<jupyter_code>[w.lower() for w in gberg_sents[4]]<jupyter_output><empty_output><jupyter_text>remove stopwords and punctuation:<jupyter_code>stpwrds = stopwords.words('english') + list(string.punctuation)
stpwrds
[w.lower() for w in gberg_sents[4] if w not in stpwrds]<jupyter_output><empty_output><jupyter_text>stem words:<jupyter_code>stemmer = PorterStemmer()
[stemmer.stem(w.lower()) for w in gberg_sents[4] if w not in stpwrds]<jupyter_output><empty_output><jupyter_text>handle bigram collocations:<jupyter_code>phrases = Phrases(gberg_sents) # train detector
bigram = Phraser(phrases) # create a more efficient Phraser object for transforming sentences
bigram.phrasegrams # output count and score of each bigram
"Jon lives in New York City".split()
bigram["Jon lives in New York City".split()]<jupyter_output><empty_output><jupyter_text>Preprocess the corpus<jupyter_code>lower_sents = []
for s in gberg_sents:
    lower_sents.append([w.lower() for w in s if w not in list(string.punctuation)])
lower_sents[0:5]
lower_bigram = Phraser(Phrases(lower_sents))
lower_bigram.phrasegrams # miss taylor, mr woodhouse, mr weston
lower_bigram["jon lives in new york city".split()]
lower_bigram = Phraser(Phrases(lower_sents, min_count=32, threshold=64))
lower_bigram.phrasegrams
# as in Maas et al. (2001):
# - leave in stop words ("indicative of sentiment")
# - no stemming ("model learns similar representations of words of the same stem when data suggests it")
clean_sents = []
for s in lower_sents:
    clean_sents.append(lower_bigram[s])
clean_sents[0:9]
clean_sents[6] # could consider removing stop words or common words<jupyter_output><empty_output><jupyter_text>Run word2vec<jupyter_code># max_vocab_size can be used instead of min_count (which has increased here)
# model = Word2Vec(sentences=clean_sents, size=64, sg=1, window=10, min_count=10, seed=42, workers=8)
# model.save('clean_gutenberg_model.w2v')<jupyter_output><empty_output><jupyter_text>Explore model<jupyter_code># skip re-training the model with the next line:  
model = gensim.models.Word2Vec.load('clean_gutenberg_model.w2v')
len(model.wv.vocab) # down from 17k in previous notebook
model['ma_am']
model.most_similar('ma_am') 
# swap woman and man
model.most_similar(positive=['ma_am', 'man'], negative=['woman']) 
model.most_similar(positive=['father', 'woman'], negative=['man'])<jupyter_output><empty_output><jupyter_text>Reduce word vector dimensionality with t-SNE<jupyter_code>tsne = TSNE(n_components=2, n_iter=1000)
X_2d = tsne.fit_transform(model[model.wv.vocab])
coords_df = pd.DataFrame(X_2d, columns=['x','y'])
coords_df['token'] = model.wv.vocab.keys()
coords_df.head()
# coords_df.to_csv('clean_gutenberg_tsne.csv', index=False)<jupyter_output><empty_output><jupyter_text>Visualise<jupyter_code>coords_df = pd.read_csv('clean_gutenberg_tsne.csv')
_ = coords_df.plot.scatter('x', 'y', figsize=(12,12), marker='.', s=10, alpha=0.2)
output_notebook()
subset_df = coords_df.sample(n=5000)
p = figure(plot_width=800, plot_height=800)
_ = p.text(x=subset_df.x, y=subset_df.y, text=subset_df.token)
show(p)
# output_file() here<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11098901098901098, 'fraction_numerical': 0.013626373626373627, 'mean_word_length': 7.871345029239766, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '55356', 'license': 'FTL', 'n_tokens_mistral': 1704, 'n_tokens_neox': 1644, 'n_words': 369, 'path': 'notebooks/natural_language_preprocessing_best_practices.ipynb', 'repo_name': 'amrendra1002/ML-DL', 'size': '4550'}