<jupyter_start><jupyter_text>1. Import Libraries<jupyter_code>import numpy as np
import pandas as pd
import os
import contractions
import nltk
nltk.download('punkt')
from nltk.tokenize import word_tokenize
nltk.download('stopwords')
from nltk.corpus import stopwords
import matplotlib.pyplot as plt
import re
import spacy
nlp = spacy.load('en_core_web_sm', disable = ['parser', 'ner'])
from sklearn.feature_extraction.text import CountVectorizer
from wordcloud import WordCloud
from gensim.models.fasttext import FastText
from sklearn.decomposition import PCA
import plotly.express as px
import pickle

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"<jupyter_output>[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.
[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.<jupyter_text>2. Read the data<jupyter_code>reddit_corpus = pd.read_csv("data/reddit_corpus_urgenta.csv")
reddit_corpus.head(2)
# Drop unnecessary columns
reddit_corpus = reddit_corpus.drop(columns = ["url", "id"])
reddit_corpus.head(2)<jupyter_output><empty_output><jupyter_text>3. Preprocess data<jupyter_code># Check data types in corpus
reddit_corpus.dtypes
# # Transform data types to strings for easy management
reddit_corpus['posts'] = reddit_corpus['posts'].astype(str)
reddit_corpus['en_titles'] = reddit_corpus['posts'].astype(str)
reddit_corpus['en_bodies'] = reddit_corpus['en_bodies'].astype(str)
reddit_corpus['title'] = reddit_corpus['title'].astype(str)
reddit_corpus['body'] = reddit_corpus['body'].astype(str)
# Plot the frequency of comments numbers
fig = reddit_corpus['comms_num'].value_counts().sort_index().plot(kind='bar')
fig.set_title('Comments number frequency')
fig.set_xlabel('Number of comments')
fig.set_ylabel('Frequency')
plt.show()
fig = reddit_corpus['word_len_posts'].value_counts().sort_index().plot(kind='bar')
fig.set_title('Frequency of number of words in posts')
fig.set_xlabel('Number of words in posts')
fig.set_ylabel('Frequency')
plt.show()
fig = reddit_corpus['word_len_en_titles'].value_counts().sort_index().plot(kind='bar')
fig.set_title('Frequency of number of words in titles')
fig.set_xlabel('Number of words in titles')
fig.set_ylabel('Frequency')
plt.show()
# Preprocess the posts text in corpus
def preprocess(document):
    document = str(document)
    # Contractions 
    document = contractions.fix(document)
    # Lowercasing
    document = document.lower()
    # Remove any whitespaces
    document = re.sub(r'[^\w\s]', ' ', document)
    document = re.sub(r'[^a-z ]', '', document)
    # Word Tokenize
    document = word_tokenize(document)
    # Stopwords
    stop_words = stopwords.words('english')
    document = [word for word in document if word not in stop_words]
    # Lemmatizer
    document = ' '.join(document)
    document = [word.lemma_ for word in nlp(document)]
    # Change the output to a sentence
    document = ' '.join(document)

    return document
reddit_corpus['posts_prep'] = [preprocess(document) for document in reddit_corpus['posts'].to_list()]
reddit_corpus.head(2)
# Save the corpus in a csv
reddit_corpus.to_csv(os.path.join("data", "reddit_corpus_cleaned.csv"), index=False)<jupyter_output><empty_output><jupyter_text>Bag of words top 5000<jupyter_code>corpus_posts = reddit_corpus['posts_prep']
# Create dtm (document term matrix) for most frequent 5000 words
vec_posts = CountVectorizer(max_features=5000)
counts_per_post = vec_posts.fit_transform(corpus_posts)
counts_per_post
posts_words = vec_posts.get_feature_names()
posts_words[1700]
dtm_posts = pd.DataFrame(counts_per_post.toarray(), columns = posts_words)
dtm_posts.head(6)
# Create new column that contains only the words in DTM (from the 5000 words)
reddit_corpus['posts_prep_lim'] = reddit_corpus['posts_prep'].apply(word_tokenize)
reddit_corpus.head(2)
# Check in the 5000 words and see if text_prep words are in there
reddit_corpus['posts_prep_lim'] = reddit_corpus['posts_prep_lim'].apply(lambda x:[word for word in x if word in posts_words])
reddit_corpus.head(2)
# Get no of words
reddit_corpus['word_len_posts_lim'] = reddit_corpus['posts_prep_lim'].apply(len)

reddit_corpus.head(2)
# Save the dtms
dtm_posts.to_csv("data/dtm_posts_bow.csv", index=False)
reddit_corpus.to_csv('data/reddit_corpus_cleaned.csv', index=False)<jupyter_output><empty_output><jupyter_text>Wordcloud<jupyter_code>wordcloud_freq_posts = dtm_posts.sum(axis=0)
wordcloud_freq_posts
for i in [100,50,20]:
    wordcloud_posts = WordCloud(max_words=i, background_color = 'black', colormap = None)
    wordcloud_posts.generate_from_frequencies(wordcloud_freq_posts)
    plt.imshow(wordcloud_posts)
    plt.axis('off')
    plt.title(f"Worcloud of top {i} words in Reddit posts")
    plt.show()<jupyter_output><empty_output><jupyter_text>Word Frequency<jupyter_code>word_freq_posts = pd.DataFrame(wordcloud_freq_posts).reset_index()
word_freq_posts.columns = ['Word', 'Count']
word_freq_posts.head()
# Let's check the most frequent words
word_freq_posts = word_freq_posts.sort_values(by='Count', ascending=False)
# Let's create a barchart of frequencies
top20_posts = word_freq_posts[0:19].copy()
top20_posts
plt.barh(y=top20_posts['Word'],width=top20_posts['Count'], height = 0.7)
plt.title("Top 20 words in posts")
plt.xlabel('Frequency')
plt.ylabel('Word')
plt.show() 
final_list = []
for word in top20_posts["Word"]:
    final_list.append(word)<jupyter_output><empty_output><jupyter_text>Text representation Using fast text<jupyter_code># Create corpus vectors from the top 5000 words in the comments and posts
corpus_posts = reddit_corpus['posts_prep_lim']
final_size = 60
window_size = 10 
min_word = 3 
corpus_posts
# Create the fast text models for both vectors
%%time
model_posts = FastText(size=final_size, 
                 window=window_size, 
                 min_count=min_word, 
                 seed=1, 
                 workers=1)
model_posts.build_vocab(sentences = corpus_posts)
model_posts.train(sentences=corpus_posts, total_examples=len(corpus_posts), epochs=50) 
# 60-dimensional vector for the word "coronavirus"
model_posts.wv['coronavirus']
model_posts.wv.most_similar('coronavirus', topn=5)
model_posts.wv.similarity(w1='coronavirus', w2='pneumonia')
# Save the model
pickle.dump(model_posts, open('data/fast_text_model_posts.pkl', 'wb'))
# Visualize the model 
X_posts = model_posts.wv[final_list]
X_posts #an array with all 60-D vectors for all the words in final_list
# To plot sth nice let's reduce the 60 dimensions to 2
pca_posts = PCA(n_components=2).fit_transform(X_posts)
pca_posts
fig = px.scatter(x = pca_posts[:,0], y = pca_posts[:,1], text = final_list)
fig.update_traces(textposition='top center', textfont_size=14)
fig.show()
# Which words tend to cluster together?<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11495584189952222, 'fraction_numerical': 0.012595917185464022, 'mean_word_length': 6.460043196544277, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '38877', 'license': 'MIT', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2383, 'n_words': 510, 'path': 'notebook/2_Reddit_PreprocessData.ipynb', 'repo_name': 'stefanacioban/covid19_sentiment_analysis', 'size': '6907'}