<jupyter_start><jupyter_text>Stress Analysis in Social MediaLeverage the newly published and labelled reddit dataset for stress analysis to develop and improve supervised learning methods for identifying stress, both neural and traditional, and analyze the complexity and diversity of the data and characteristics of each category.<jupyter_code>import pandas as pd
import numpy as np 
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import seaborn as sns
import re
import string

import nltk
from nltk.corpus import stopwords
from nltk.corpus import RegexpTokenizer
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, f1_score
from sklearn.metrics import precision_recall_fscore_support
from sklearn.model_selection import train_test_split
from sklearn.manifold import TSNE
import xgboost as xgb
from wordcloud import WordCloud<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>path = '../../data/'
train = pd.read_pickle(path + "train.pkl")
test = pd.read_pickle(path + "test.pkl")

domain = {'domesticviolence':'abuse',
          'survivorsofabuse':'abuse',
          'anxiety': 'anxiety',
          'stress': 'anxiety', 
          'almosthomeless': 'financial',
          'assistance': 'financial',
          'food_pantry': 'financial', 
          'homeless': 'financial',
          'ptsd': 'ptsd',
          'relationships': 'social'}

train['domain'] = train['subreddit'].map(domain)
test['domain'] = test['subreddit'].map(domain)
print('Number of train data', len(train))
sns.countplot('domain', data = train, palette='PRGn')
print('Number of train data', len(train))
train['label'].value_counts()
print('Number of train data', len(test))
test['label'].value_counts()<jupyter_output>Number of train data 715<jupyter_text>When doing the presentation, can add a part of analyzing the text with lexicon-based tool such as textblob  Data PreprocessingPossible approach of data preprocessing and feature engineeing - Tokenization, Removal of punctuations.- Removal of commonly used words (stopwords).- Lemmatization or Stemming (Normalization)  Tokenization & Remove punctuationsremove punctuations, single letter word, word that contains number...<jupyter_code># tokenization: breaks the string into list of words
tokenizer = RegexpTokenizer(r'[a-zA-Z]{2,}')
train['processed_text'] = train['text'].apply(lambda x: tokenizer.tokenize(x.lower()))
test['processed_text'] = test['text'].apply(lambda x: tokenizer.tokenize(x.lower()))<jupyter_output><empty_output><jupyter_text>Remove Stopwords<jupyter_code>from tqdm import tqdm
nltk.download('stopwords')
def remove_stopwords(text):
    words = [w for w in text if w not in stopwords.words('english')]
    return words
tqdm.pandas()
train['processed_text'] = train['processed_text'].progress_apply(lambda x: remove_stopwords(x))
test['processed_text'] = test['processed_text'].apply(lambda x: remove_stopwords(x))
train['processed_text'][:5]<jupyter_output><empty_output><jupyter_text>Stemming & LemmatizationText normalization<jupyter_code># stemming
stemmer = PorterStemmer()

def word_stemmer(text):
    stem_text = [stemmer.stem(i) for i in text]
    return stem_text

train['processed_text'] = train['processed_text'].apply(lambda x: word_stemmer(x))
test['processed_text'] = test['processed_text'].apply(lambda x: word_stemmer(x))
nltk.download('wordnet')
# lemmatization
lemmatizer = WordNetLemmatizer()

def word_lemmatizer(text):
    lem_text = [lemmatizer.lemmatize(i) for i in text]
    return lem_text

train['processed_text'] = train['processed_text'].apply(lambda x: word_lemmatizer(x))
test['processed_text'] = test['processed_text'].apply(lambda x: word_lemmatizer(x))
path = '../../data/preprocessed/'
train = pd.read_pickle(path + "train.pkl")
test = pd.read_pickle(path + "test.pkl")
def create_word_cloud(all_words):
    wordcloud = WordCloud(width=800, height=500, random_state=21, max_font_size=110).generate(all_words)
    plt.figure(figsize=(10, 7))
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis('off')
    plt.show()
    plt.savefig('../../figs/stress_wordcloud.png')

train['processed_text'] = train['processed_text'].apply(lambda x: ' '.join(x))
non_stress_post_words = ' '.join([text for text in train[train['label'] == 0]['processed_text']])
stress_post_words = ' '.join([text for text in train[train['label'] == 1]['processed_text']])

# create_word_cloud(non_stress_post_words)
create_word_cloud(stress_post_words)

train['text'].iloc[0]
train['processed_text'].iloc[0]<jupyter_output><empty_output><jupyter_text>Feature Extraction- vectorization- bag-of-n-grams- tf-idf- word2vec- word embedding (pretrained- fasttext - glove   TF-IDFFind TF-IDF of unigram, bigram and trigram<jupyter_code>path = '../../data/preprocessed/'
train = pd.read_pickle(path + "train.pkl")
test = pd.read_pickle(path + "test.pkl")
train = train[train['confidence']>0.8]
test = test[test['confidence']>0.8]
# train_corpus = train['processed_text']
train_corpus = train['processed_text'].apply(lambda x: ' '.join(x))
# test_corpus = test['processed_text']
test_corpus = test['processed_text'].apply(lambda x: ' '.join(x))
X = train_corpus.append(test_corpus)
X = X.reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Unigram<jupyter_code>vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1,1)) #bigram
X_new = vectorizer.fit_transform(X)

X_train = X_new.toarray()[:len(train)]
X_test = X_new.toarray()[len(train):]
print(X_train.shape)
print(X_test.shape)
def top_tfidf_features(row, features, top_n):
    ''' Get top n tfidf values in row and return them with their corresponding feature names.'''
    topn_ids = np.argsort(row)[::-1][:top_n]
    top_features = [(features[i], row[i]) for i in topn_ids]
    df = pd.DataFrame(top_features)
    df.columns = ['feature', 'tfidf']
    return df

def top_features_in_post(data, features, row_id, top_n):
    ''' Top tfidf features in specific document (matrix row) '''
    row = np.squeeze(data[row_id])
    return top_tfidf_features(row, features, top_n)
features = vectorizer.get_feature_names()
top_features_in_post(X_train, features, 0, 10)
def top_mean_features_by_label(X, y, features, top_n, label_id):
    ids = np.where(y == label_id)
    
    if ids:
        new_data = X[ids]
    else:
        new_data = X


    tfidf_means = np.mean(new_data, axis=0)
    feature_df = top_tfidf_features(tfidf_means, features, top_n)
    feature_df['label'] = label_id

    return feature_df    
non_stress_df = top_mean_features_by_label(X_train, train['label'], features, 200, 0)
stress_df = top_mean_features_by_label(X_train, train['label'], features, 200, 1)
fig, ax = plt.subplots(1, 2, figsize=(12, 7))
non_stress_df[['feature', 'tfidf']][:20].plot(kind='barh', ax=ax[0])
stress_df[['feature', 'tfidf']][:20].plot(kind='barh', ax=ax[1])

ax[0].set_title('Non-stress Posts')
ax[0].set_xlabel('weights')
ax[0].set_yticklabels(non_stress_df.feature)

ax[1].set_title('Stress Posts')
ax[1].set_xlabel('weights')
ax[1].set_yticklabels(stress_df.feature)


fig.suptitle('TF-IDF Top 20 words')
plt.subplots_adjust(wspace = 0.5)
plt.show()
plt.savefig('../../figs/Top_20_words.png')
def create_tfidf_word_cloud(weights):
    wc = WordCloud(width=800, height=500, random_state=21, max_font_size=110)
    wordcloud = wc.generate_from_frequencies(weights)
    
    plt.figure(figsize=(10, 7))
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis('off')
    plt.show()
non_stress_weights = dict(zip(non_stress_df.feature, non_stress_df.tfidf))
create_tfidf_word_cloud(non_stress_weights)
plt.savefig('../../figs/non_stress_word_cloud.png')
stress_weights = dict(zip(stress_df.feature, stress_df.tfidf))
create_tfidf_word_cloud(stress_weights)
plt.savefig('../../figs/stress_word_cloud.png')<jupyter_output><empty_output><jupyter_text>Bigram<jupyter_code>vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1,2)) #bigram
X_new = vectorizer.fit_transform(X)
X_train = X_new.toarray()[:len(train)]
X_test = X_new.toarray()[len(train):]
X_train = pd.DataFrame(X_train)
X_test = pd.DataFrame(X_test)
print(X_train.shape)
print(X_test.shape)
X_train.to_pickle(path+ "bigram_train.pkl")
X_test.to_pickle(path + "bigram_test.pkl")<jupyter_output><empty_output><jupyter_text>Classification  Unigram<jupyter_code>y_train = train['label']
y_test = test['label']

clf_dict = {'log reg': LogisticRegression(random_state=42), 
            'naive bayes': GaussianNB(), 
            'linear svc': LinearSVC(random_state=42),
            'CART': DecisionTreeClassifier(random_state=42),
            'random forest': RandomForestClassifier(n_estimators=100, random_state=42)}
for name, clf in clf_dict.items():
    model = clf.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    print('Accuracy and F1-score of {}:'.format(name), accuracy_score(y_pred, y_test), f1_score(y_test, y_pred))<jupyter_output>Accuracy and F1-score of log reg: 0.8422818791946308 0.8645533141210374
Accuracy and F1-score of naive bayes: 0.6208053691275168 0.6705539358600582
Accuracy and F1-score of linear svc: 0.8355704697986577 0.8571428571428572
Accuracy and F1-score of CART: 0.6912751677852349 0.7371428571428572
Accuracy and F1-score of random forest: 0.7684563758389261 0.8109589041095889<jupyter_text>XGBoost<jupyter_code># on CPU:
D_train = xgb.DMatrix(X_train, label=y_train)
D_test = xgb.DMatrix(X_test, label=y_test)
params = {"objective":'multi:softprob','colsample_bytree': 0.3, 'learning_rate': 0.1,
                'max_depth': 5, 'alpha': 10 , 'num_class': 2, 'random_state': 42}
steps = 20  # The number of training iterations
model = xgb.train(params, D_train, steps)
preds = model.predict(D_test)
y_pred = np.asarray([np.argmax(line) for line in preds])
print('Accuracy', accuracy_score(y_pred, y_test))
print('Precision', precision_score(y_test, y_pred))
print('Recall', recall_score(y_test, y_pred))
print('F1-Score', f1_score(y_test, y_pred))<jupyter_output>Accuracy 0.7315436241610739
Precision 0.7654320987654321
Recall 0.7469879518072289
F1-Score 0.7560975609756098<jupyter_text>Logistic Regression<jupyter_code>clf = LogisticRegression(random_state=0).fit(X_train, y_train)
y_pred = clf.predict(X_test)
print('Accuracy', accuracy_score(y_pred, y_test))
print('Precision', precision_score(y_test, y_pred))
print('Recall', recall_score(y_test, y_pred))
print('F1-Score', f1_score(y_test, y_pred))
# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)

df_cm = pd.DataFrame(cm, index = [i for i in "01"], columns = [i for i in "01"])
plt.figure(figsize = (5, 3))
sns.heatmap(df_cm, annot=True, cmap='Blues', fmt='g')<jupyter_output><empty_output><jupyter_text>Bigram<jupyter_code>path = '../../data/preprocessed/'
X_train = pd.read_pickle(path + "bigram_train.pkl")
X_test = pd.read_pickle(path + "bigram_test.pkl")
for name, clf in clf_dict.items():
    model = clf.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    print('Accuracy and F1-score of {}'.format(name), accuracy_score(y_pred, y_test), f1_score(y_test, y_pred))
clf = LinearSVC(random_state=42).fit(X_train, y_train)
y_pred = clf.predict(X_test)
accuracy_score(y_test, y_pred)
# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)

df_cm = pd.DataFrame(cm, index = [i for i in "01"], columns = [i for i in "01"])
plt.figure(figsize = (5, 3))
sns.heatmap(df_cm, annot=True, cmap='Blues', fmt='g')
print('Accuracy', accuracy_score(y_pred, y_test))
print('Precision', precision_score(y_test, y_pred))
print('Recall', recall_score(y_test, y_pred))
print('F1-Score', f1_score(y_test, y_pred))<jupyter_output>Accuracy 0.8422818791946308
Precision 0.8324022346368715
Recall 0.8975903614457831
F1-Score 0.863768115942029<jupyter_text>xgboost<jupyter_code># on CPU:
D_train = xgb.DMatrix(X_train, label=y_train)
D_test = xgb.DMatrix(X_test, label=y_test)
params = {"objective":'multi:softprob','colsample_bytree': 0.3, 'learning_rate': 0.1,
                'max_depth': 5, 'alpha': 10 , 'num_class': 2, 'random_state': 42}
steps = 20  # The number of training iterations
model = xgb.train(params, D_train, steps)
preds = model.predict(D_test)
y_pred = np.asarray([np.argmax(line) for line in preds])
print('Accuracy', accuracy_score(y_pred, y_test))
print('Precision', precision_score(y_test, y_pred))
print('Recall', recall_score(y_test, y_pred))
print('F1-Score', f1_score(y_test, y_pred))<jupyter_output>Accuracy 0.7013422818791947
Precision 0.7452229299363057
Recall 0.7048192771084337
F1-Score 0.7244582043343653
{'fraction_non_alphanumeric': 0.13590216889709275, 'fraction_numerical': 0.04045531456698969, 'mean_word_length': 6.400682982356289, 'pattern_counts': {'":': 2, '<': 48, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '292706', 'license': 'MIT', 'n_tokens_mistral': 5152, 'n_tokens_neox': 4701, 'n_words': 929, 'path': 'src/notebooks/train/tf_idf.ipynb', 'repo_name': 'Evaaaaaaa/Learn2Relax-Stress-Detection-on-Social-Media', 'size': '13002'}