<jupyter_start><jupyter_text>EDA on tweets  Distribution of sentiments in tweets- Most of the tweets are neutral tweets- The average sentiment of the tweets is 0.058 (mostly neutral, slightly on the positive side)<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
sns.distplot(df_train.sentiment, bins=10)
plt.xlim((-1,1))
plt.title('Histogram distribution of sentiment')
plt.show()
df_train.sentiment.describe()<jupyter_output><empty_output><jupyter_text>Word cloud for positive and negative tweets<jupyter_code># Separate the train dataset into positive and negative sentiment
threshold = 0.3
neg_df = df_train.loc[df_train.sentiment < -threshold, :]
pos_df = df_train.loc[df_train.sentiment > threshold, :]
pos_corpus = pos_df.tweet.values
neg_corpus = neg_df.tweet.values
total_corpus = df_train.tweet.values
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
vectorizer.fit(total_corpus)
import numpy as np
pos_vec = vectorizer.transform(pos_corpus)
neg_vec = vectorizer.transform(neg_corpus)
diff_vec = np.sum(pos_vec, axis=0) - np.sum(neg_vec, axis=0)
diff_df = pd.DataFrame(diff_vec, columns = vectorizer.get_feature_names()).T
threshold = 50
pos_wc = diff_df.loc[(diff_df[0] > threshold)]
neg_wc = diff_df.loc[(diff_df[0] < -threshold)]
print('Number of words in pos_wc: ' + str(len(pos_wc)))
print('Number of words in neg_wc: ' + str(len(neg_wc)))
pos_tokens = []
for i,feq in enumerate(pos_wc[0]):
    for t in range(feq):
        pos_tokens.append(pos_wc.index[i])
pos_tokens = ' '.join(pos_tokens)

neg_tokens = []
for i,feq in enumerate(neg_wc[0]):
    feq = -feq
    for t in range(feq):
        neg_tokens.append(neg_wc.index[i])
neg_tokens = ' '.join(neg_tokens)
from wordcloud import WordCloud, ImageColorGenerator

pos_wordcloud = WordCloud(max_font_size=50, max_words=50, collocations=False).generate(pos_tokens)
neg_wordcloud = WordCloud(max_font_size=50, max_words=50, collocations=False).generate(neg_tokens)

plt.imshow(pos_wordcloud, interpolation='bilinear')
plt.title('Positive Sentiment Wordcloud')
plt.axis("off")
plt.show()

plt.imshow(neg_wordcloud, interpolation='bilinear')
plt.title('Negative Sentiment Wordcloud')
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Wordcloud for common words between negative and positive tweets<jupyter_code>pos_count = np.sum(pos_vec, axis=0)
neg_count = np.sum(neg_vec, axis=0)
common_vec = np.sqrt(np.multiply(pos_count, neg_count))
common_df = pd.DataFrame(common_vec, columns = vectorizer.get_feature_names()).T
threshold = 200
common_wc = common_df.loc[(common_df[0] > threshold)]

print('Number of words in common_wc: ' + str(len(common_wc)))
common_tokens = []
for i,feq in enumerate(common_wc[0]):
    for t in range(int(feq)):
        common_tokens.append(common_wc.index[i])
common_tokens = ' '.join(common_tokens)
common_wordcloud = WordCloud(max_font_size=50, max_words=50, collocations=False).generate(common_tokens)

plt.imshow(common_wordcloud, interpolation='bilinear')
plt.title('Common words in both sentiments')
plt.axis("off")
plt.show()
pos_df = pd.DataFrame(pos_count, columns = vectorizer.get_feature_names()).T
neg_df = pd.DataFrame(neg_count, columns = vectorizer.get_feature_names()).T
# Corona and virus appear in both 'common' and 'positive' cloud
# because these words appear in high frequency in both sentiment
# but it appeared more in positive tweet.

word = 'corona'

print('common_df: ' + str(common_df.loc[word].values))
print('pos_df: ' + str(pos_df.loc[word].values))
print('neg_df: ' + str(neg_df.loc[word].values))
print('diff_df: ' + str(diff_df.loc[word].values))<jupyter_output>common_df: [11372.27281593]
pos_df: [16553]
neg_df: [7813]
diff_df: [8740]<jupyter_text>Length of tweets and sentiment<jupyter_code>df_train['tweet_len'] = df_train.tweet.apply(len) 
threshold = 0.3
df_train['class'] = 'neutral'
df_train.loc[df_train.sentiment > threshold ,['class']] = 'pos'
df_train.loc[df_train.sentiment < -threshold ,['class']] = 'neg'
sns.scatterplot(data=df_train, x='sentiment', y='tweet_len', alpha=0.7, hue='class')
plt.ylabel('Length of tweet')
plt.show()
sns.violinplot(data=df_train, x='class', y='tweet_len')
plt.ylabel('Length of tweet')
plt.xlabel('Sentiment of tweet')
plt.show()
import re

def find_hashtag(entry):
    hashtag_pattern = r'#.+?\s'
    output = re.findall(hashtag_pattern, entry)
    if len(output) == 0:
        output = None
    return output

df_train['hashtag'] = df_train.tweet.apply(find_hashtag)
df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 137165 entries, 68740 to 53264
Data columns (total 7 columns):
 #   Column        Non-Null Count   Dtype  
---  ------        --------------   -----  
 0   created_date  137165 non-null  object 
 1   tweet         137165 non-null  object 
 2   tweet_id      137165 non-null  int64  
 3   sentiment     137165 non-null  float64
 4   tweet_len     137165 non-null  int64  
 5   class         137165 non-null  object 
 6   hashtag       0 non-null       object 
dtypes: float64(1), int64(2), object(4)
memory usage: 13.4+ MB<jupyter_text>Vocabulary analysis<jupyter_code>sns.distplot(common_df)
plt.show()
common_df.describe()
unique_words = common_df.loc[(common_df[0] == 0)]
print('Number of words unique to the respective sentiment: ' + str(len(unique_words)))

uni_words = list(unique_words.index)
import pickle
with open("data/unique_word.txt", "wb") as fp:
    pickle.dump(uni_words, fp)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1318840579710145, 'fraction_numerical': 0.026992753623188405, 'mean_word_length': 5.487661574618096, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '84985', 'license': 'MIT', 'n_tokens_mistral': 2104, 'n_tokens_neox': 1952, 'n_words': 420, 'path': 'notebooks/eda.ipynb', 'repo_name': 'dashtagger/sentiment_analysis_covid', 'size': '5520'}