<jupyter_start><jupyter_text>Setup and Imports<jupyter_code>import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
%matplotlib inline
import seaborn as sns
import re
sns.set
import spacy
import nltk
from nltk.corpus import brown
from nltk.corpus import wordnet
from collections import Counter
import Tweet_Normalizer as tn
from sklearn.manifold import TSNE
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.cluster import KMeans<jupyter_output><empty_output><jupyter_text>Load the Data<jupyter_code>tweets = pd.read_csv("data/train.csv")
#View first few rows
pd.set_option('display.max_colwidth', 2)
tweets.head()
#Check the dimensions
tweets.shape<jupyter_output><empty_output><jupyter_text>Clean and Normalize Tweets<jupyter_code>%%time
#USe tweet scrubber function to clean the data
tweets = tn.tweet_scrubber(tweets, verbose = True)
#Check for blank rows after cleaning. We expect 5
tweets = tweets.replace(r'^(\s)+$', np.nan, regex = True)
tweets.isnull().sum()
#Look at which rows are missing
tweets.loc[tweets["Clean Tweets"].isnull()]
#Drop the empty rows
tweets.dropna(subset=["Clean Tweets"], inplace = True)
#Double check that the missing rows were actually dropped
tweets = tweets.replace(r'^(\s)+$', np.nan, regex = True)
tweets.isnull().sum()
#Reset the index in place
tweets.reset_index(drop = True, inplace = True)
#Take a look at the last few rows of the data
tweets.tail(n=10)<jupyter_output><empty_output><jupyter_text>Create Functions to Examine Tweet Details<jupyter_code>def word_count(text):
    """
    This function counts the total words in the cleaned tweet
    
    test: String containing the text
    
    returns: Length of the text as an int

    """
    
    text = tn.word_tokenizer(text)
    
    return len(text)
nlp = spacy.load('en_core_web_sm')
def POS_counter(text, POS):
    """
    This function counts the POS tags in the text corpus and returns the total count
    
    text: String corpus
    
    returns: Number of POS in the tweet as int

    """
    text = nlp(text)
    POS_Count = 0
    
    
    for word in text:
        if word.pos_ == POS:
            POS_Count += 1
            
    return POS_Count
#Get the length of the clean tweets
tweets["Count"] = tweets["Clean Tweets"].apply(word_count)
%%time
#Get the count and proportion of the four main parts of speech (POS)
POS_lst = ["NOUN", "VERB", "ADJ", "ADV"]

for pos in POS_lst:
    tweets[pos] = tweets["Clean Tweets"].apply(POS_counter, POS = pos)
    pos_prop = pos + "_prop"
    tweets[pos_prop] = tweets[pos] / tweets["Count"]
#Explicity label the tweet as disaster or non-disaster(safe)
Tweet = ["Safe" if tweet == 0 else "Disaster" for tweet in tweets['target']]
#Add a new column based on target
tweets["Tweet"] = Tweet
tweets.head()<jupyter_output><empty_output><jupyter_text>Data Visualizations  Percentage of Disaster and Safe Tweets<jupyter_code>twt_df = pd.DataFrame(tweets["Tweet"].value_counts() / tweets.shape[0]).reset_index().rename(columns={"index": "Tweet", "Tweet": "Percentage"})
twt_df
sns.set(rc = {'figure.figsize':(10,10)})
sns.barplot(x="Tweet", y = "Percentage", data = twt_df, color = "teal")
plt.title("Tweet Type Percentage")
plt.ylabel("Percentage")
plt.xlabel("Tweet Type")<jupyter_output><empty_output><jupyter_text>Analyze Word Occurence<jupyter_code># Create a list of all words used in safe tweets
words = [word for row in tweets["Clean Tweets"] for word in row.split()]
# Create a counter to count the words
word_cnt = Counter(words)
# Create a dictionary of the words
word_dict = dict(word_cnt.most_common())
# Create a dataframe with words and their count
word_df = pd.DataFrame({"Word": word_dict.keys(), "Count": word_dict.values()})
# Create a new column with the word percentage
word_df["Percentage"] = (word_df["Count"] / len(words)) * 100
word_df.head(n=10)
#Plot the top 30 words ranked by their percentage
sns.set(rc = {'figure.figsize':(12,10)})
sns.barplot(x="Percentage", y="Word", data = word_df[:30], color = "teal")
plt.title("Top 30 Words By Percentage")<jupyter_output><empty_output><jupyter_text>Analyze the distribution of word counts<jupyter_code>word_grp_df = pd.DataFrame(word_df["Count"]\
                                           .value_counts()\
                                           .reset_index()\
                                           .rename(columns={"index": "Word Occurrence", "Count" :"Number of Words"}))\
                                            .sort_values(by="Word Occurrence")
word_grp_df[["Number of Words", "Word Occurrence"]]<jupyter_output><empty_output><jupyter_text>We see that the majority of words, 3,257, occurr are seen only once in the tweets. Let's take a look at what kinds of words these are.<jupyter_code>word_df.loc[word_df["Count"] == 1].head(n=20)<jupyter_output><empty_output><jupyter_text>Analyze Word Occurence in Safe Tweets  Analyze the most common words<jupyter_code># Create a list of all words used in safe tweets
safe_words = [word for row in tweets.loc[tweets["target"] == 0, "Clean Tweets"] for word in row.split()]
# Create a counter to count the words
safe_cnt = Counter(safe_words)
# Create a dictionary of the top 30 words
safe_dict = dict(safe_cnt.most_common())
# Create a dataframe with words and their count
safe_df = pd.DataFrame({"Word": safe_dict.keys(), "Count": safe_dict.values()})
# Create a new column with the word percentage
safe_df["Percentage"] = (safe_df["Count"] / len(safe_words)) * 100
safe_df.head(n=10)
#Plot the top 30 words ranked by their percentage
sns.set(rc = {'figure.figsize':(12,10)})
sns.barplot(x="Percentage", y="Word", data = safe_df[:30], color = "teal")
plt.title("Top 30 Words By Percentage in Safe Tweets")<jupyter_output><empty_output><jupyter_text>Analyze Word Occurence in Disaster Tweets  Analyze the most common words<jupyter_code>disaster_words = [word for row in tweets.loc[tweets["target"] == 1, "Clean Tweets"] for word in row.split()]
disaster_cnt = Counter(disaster_words)
disaster_dict = dict(disaster_cnt.most_common())
disaster_df = pd.DataFrame({"Word": disaster_dict.keys(), "Count": disaster_dict.values()})
disaster_df["Percentage"] = (disaster_df["Count"] / len(disaster_words)) * 100
disaster_df.head(n = 10)
#Plot the top 30 words ranked by their percentage
sns.set(rc = {'figure.figsize':(12,10)})
sns.barplot(x="Percentage", y="Word", data = disaster_df[:30], color = "teal")
plt.title("Top 30 Words By Percentage in Disaster Tweets")<jupyter_output><empty_output><jupyter_text>We see that the top few words for both safe and disaster tweets tend to be the same. However, they drastically change into words we'd expect to find in a disaster tweet and a safe tweet.   Analyze Tweet Length<jupyter_code>twt_grp = tweets.groupby("Tweet")["Count"].aggregate(["mean", "std"]).sort_values(by="mean", ascending=False)
twt_grp
#Make a smoothed histogram showing the distribution of the number of words per tweet by tweet type
sns.set(rc = {'figure.figsize':(12,10)})
sns.kdeplot(tweets.loc[tweets["target"] == 0, "Count"], shade=True, color = 'blue')
sns.kdeplot(tweets.loc[tweets["target"] == 1, "Count"], shade=True, color = 'red')
plt.legend(["Safe", "Disaster"], frameon=False, fontsize='small')
plt.xlabel("Length of Tweet")
plt.title("Length of Tweet By Tweet Type")<jupyter_output><empty_output><jupyter_text>We can see that on average, disaster tweets use more words per tweet than safe tweets.   Analyze Tweet Parts of Speech To analyze the POS in each tweet, we'll look at the proportion by tweet. Therefore, we'll get an understanding on the POS structure for safe and disaster tweets.   Nouns<jupyter_code>twt_noun_grp = tweets.groupby("Tweet")["NOUN_prop"].aggregate(["mean", "std"]).sort_values(by="mean", ascending=False)
twt_noun_grp
#Make a smoothed histogram showing the distribution of the number of words per tweet by tweet type
sns.set(rc = {'figure.figsize':(12,10)})
sns.kdeplot(tweets.loc[tweets["target"] == 0, "NOUN_prop"], shade=True, color = 'blue')
sns.kdeplot(tweets.loc[tweets["target"] == 1, "NOUN_prop"], shade=True, color = 'red')
plt.legend(["Safe", "Disaster"], frameon=False, fontsize='small')
plt.xlabel("Proportion of Nouns")
plt.title("Proportion of Nouns By Tweet Type")<jupyter_output><empty_output><jupyter_text>Verbs<jupyter_code>twt_verb_grp = tweets.groupby("Tweet")["VERB_prop"].aggregate(["mean", "std"]).sort_values(by="mean", ascending=False)
twt_verb_grp
#Make a smoothed histogram showing the distribution of the number of words per tweet by tweet type
sns.set(rc = {'figure.figsize':(12,10)})
sns.kdeplot(tweets.loc[tweets["target"] == 0, "VERB_prop"], shade=True, color = 'blue')
sns.kdeplot(tweets.loc[tweets["target"] == 1, "VERB_prop"], shade=True, color = 'red')
plt.legend(["Safe", "Disaster"], frameon=False, fontsize='small')
plt.xlabel("Proportion of Verbs")
plt.title("Proportion of Verbs By Tweet Type")<jupyter_output><empty_output><jupyter_text>Adjectives<jupyter_code>twt_verb_grp = tweets.groupby("Tweet")["ADJ_prop"].aggregate(["mean", "std"]).sort_values(by="mean", ascending=False)
twt_verb_grp
#Make a smoothed histogram showing the distribution of the number of words per tweet by tweet type
sns.set(rc = {'figure.figsize':(12,10)})
sns.kdeplot(tweets.loc[tweets["target"] == 0, "ADJ_prop"], shade=True, color = 'blue')
sns.kdeplot(tweets.loc[tweets["target"] == 1, "ADJ_prop"], shade=True, color = 'red')
plt.legend(["Safe", "Disaster"], frameon=False, fontsize='small')
plt.xlabel("Proportion of Adjectives")
plt.title("Proportion of Adjective By Tweet Type")<jupyter_output><empty_output><jupyter_text>Adverbs<jupyter_code>twt_verb_grp = tweets.groupby("Tweet")["ADV_prop"].aggregate(["mean", "std"]).sort_values(by="mean", ascending=False)
twt_verb_grp
#Make a smoothed histogram showing the distribution of the number of words per tweet by tweet type
sns.set(rc = {'figure.figsize':(12,10)})
sns.kdeplot(tweets.loc[tweets["target"] == 0, "ADV_prop"], shade=True, color = 'blue')
sns.kdeplot(tweets.loc[tweets["target"] == 1, "ADV_prop"], shade=True, color = 'red')
plt.legend(["Safe", "Disaster"], frameon=False, fontsize='small')
plt.xlabel("Proportion of Adverbs")
plt.title("Proportion of Adverbs By Tweet Type")<jupyter_output><empty_output><jupyter_text>POS Comments We see that safe and disaster tweets differ in the proportion of nouns and verbs that make up their tweets. Disaster tweets tend to have more nouns. Intuitively, this makes sense as disasters are typically things that take place at a location occur to some object. On the other hand, safe tweets are made up of slightly more verbs and have twice as many adverbs than disaster tweets. Finally, both tweets tend to have the same amount of adjectives.   Tweet Visualization Using t-SNE<jupyter_code>cv = CountVectorizer(ngram_range=(2,2), min_df = 0.0, max_df = 1.0)
t_bow = cv.fit_transform(tweets["Clean Tweets"])
cv = CountVectorizer(ngram_range=(1,1), min_df = 0.0, max_df = 1.0)
t_bow = cv.fit_transform(tweets["Clean Tweets"])
tsne = TSNE(perplexity=50)
tsvd = TruncatedSVD(n_components=50)
t_50D = tsvd.fit_transform(t_bow)
t_2D = tsne.fit_transform(t_50D)
sns.set(rc = {'figure.figsize':(12,10)})
sns.scatterplot(x=t_2D[:, 0], y=t_2D[:, 1], hue = tweets["Tweet"])
plt.title("t-SNE Visualization of Tweets")<jupyter_output><empty_output><jupyter_text>Find the best t-SNE representation based on KMeans<jupyter_code>%%time
cv = CountVectorizer(ngram_range=(1,1), min_df = 0.0, max_df = 1.0)
t_bow = cv.fit_transform(tweets["Clean Tweets"])

inertia_results = []
svd_n_comp = []
perp = []
n_iter = 100
count = 1

for i in range(n_iter):
    perplexity = np.random.randint(5, 100)
    perp.append(perplexity)
    n_components = np.random.randint(25, 100)
    svd_n_comp.append(n_components)
    
    tsne = TSNE(perplexity=perplexity)
    tsvd = TruncatedSVD(n_components=n_components)
    
    t_50D = tsvd.fit_transform(t_bow)
    t_2D = tsne.fit_transform(t_50D)
    
    kmeans = KMeans(n_clusters = 2)
    kmeans.fit_predict(t_2D)
    inertia_results.append(kmeans.inertia_)
    if count % 10 == 0:
        print(f"Done with {count} of {n_iter}")
    count += 1
    
tsne_df = pd.DataFrame({"KMeans Inertia": inertia_results, "t-SNE Perplexity": perp, "SVD Components": svd_n_comp})
tsne_df.sort_values(by="KMeans Inertia").head()
min_inert = tsne_df["KMeans Inertia"].argmin()
best_perp = tsne_df.loc[min_inert, "t-SNE Perplexity"]
best_comp = tsne_df.loc[min_inert, "SVD Components"]
tsne = TSNE(perplexity=best_perp)
tsvd = TruncatedSVD(n_components=best_comp)
    
t_50D = tsvd.fit_transform(t_bow)
t_2D = tsne.fit_transform(t_50D)
sns.set(rc = {'figure.figsize':(12,10)})
sns.scatterplot(x=t_2D[:, 0], y=t_2D[:, 1], hue = tweets["Tweet"])
plt.title("t-SNE Visualization of Tweets")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13368609446735663, 'fraction_numerical': 0.013228542525873473, 'mean_word_length': 5.477822580645161, 'pattern_counts': {'":': 12, '<': 69, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '608861', 'license': 'MIT', 'n_tokens_mistral': 4786, 'n_tokens_neox': 4498, 'n_words': 1190, 'path': 'Tweet Visualization.ipynb', 'repo_name': 'AustinYunker/Twitter-Prediction', 'size': '12851'}