<jupyter_start><jupyter_text>SENTIMENT ANALYSIS FOR DEPRESSION DETECTION   DATA EXTRACTION<jupyter_code>import tweepy
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
import api_keys
import json
auth=OAuthHandler(api_keys.consumer_key,api_keys.consumer_secret)
auth.set_access_token(api_keys.access_token,api_keys.access_secret)
api=tweepy.API(auth)
class std_listener(StreamListener):
    def on_data(self, data):
        with open('tweets_file.txt','a') as fp:
            fp.write(data)
            
        return True

    def on_error(self, status):
        print(status)
var=std_listener()
stream=Stream(auth,var)
stream.filter(track=['I\'m fine','I want to die','No one cares','I am sad','anxiety','suicide','I hate my life','I want to end my life','I feel sad','tired','helpless','hopeless','aching','lost','worthless','useless','stupid','stuck','adrift','hurting','alone','unsure','insecure','despair','I should','stressed'],languages=['en'])<jupyter_output><empty_output><jupyter_text>PRE-PROCESSING<jupyter_code>import pandas as pd
import json
import re
import string
import unicodedata
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
raw_tweets_text=[]

def get_data(url):
    with open(url) as fp:
        for line in fp:
                if(line!="\n"):
                    tweet_as_dict=json.loads(line)
                    raw_tweets_text.append(tweet_as_dict['text'])
    convert_to_csv()

def convert_to_csv():    
    df=pd.DataFrame(raw_tweets_text)
    df.to_csv('tweets.csv')
get_data('tweets_file.txt')
d = pd.read_csv("tweets.csv")
d = d.drop("Unnamed: 0",axis=1)
d = d.rename(columns={"0":"Text"})
d.drop_duplicates(inplace=True)
d = d.reset_index(drop=True)
d.to_csv('Tweets.csv')
d.head(10)
d.info()
def process_data(df):
    text_emoji = re.compile('[\U00010000-\U0010ffff]', flags=re.UNICODE)
    text_links = re.compile('httpstco[a-zA-Z0-9]+')
    text_rts = re.compile('RT [A-Za-z0-9]+')
    text_schar = re.compile('[^A-Za-z0-9 ]+')
    text_nums = re.compile('[0-9]+')
    
    for i in range(len(df)):
        txt=df["Text"].iloc[i]
        txt=text_emoji.sub(r'',txt)
        temp=txt.translate(txt.maketrans('','',string.punctuation))
        temp=text_links.sub(r'',temp)
        temp=text_rts.sub(r'',temp)        
        temp=text_schar.sub(r'',temp)
        temp=text_nums.sub(r'',temp)
        temp = unicodedata.normalize('NFKD',temp).encode('ascii', 'ignore').decode('utf-8', 'ignore')
        df["Text"].iloc[i] = temp.lower()
    return df   
df = process_data(d.copy())
df.head(10)
def further_ops(df):
    for i in range(len(df)):
        tweet = df["Text"].iloc[i]
        token = word_tokenize(tweet)
        token_no_sw = [word for word in token if word not in stopwords.words()]
        lemmatizer = WordNetLemmatizer()
        lemm_token = [lemmatizer.lemmatize(word) for word in token_no_sw]
        df["Text"].iloc[i] = " ".join(lemm_token)
    return df    
dft = further_ops(df.copy())
dft.head(10)
index = dft[dft["Text"]==""].index
dft.drop(index,inplace=True)
d = dft.reset_index(drop="True")
d.info()
d.to_csv('processed_tweets.csv')<jupyter_output><empty_output><jupyter_text>FEATURE EXTRACTION<jupyter_code>import pandas as pd
from textblob import TextBlob

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
d = pd.read_csv("processed_tweets.csv")
d = d.drop(["Unnamed: 0"],axis=1)
d.head(10)
d.drop_duplicates(inplace=True)
d.isna().sum()
d.dropna(inplace=True)
d.info()
n = len(d)
d["Sentiment"]=[None for i in range(n)]
for i in range(n):
    
    s = TextBlob(d["Text"].iloc[i]).sentiment
    if(s[0]>=0):
        d["Sentiment"].iloc[i] = 0
    else:
        d["Sentiment"].iloc[i] = 1
len(d[d["Sentiment"]==0]), len(d[d["Sentiment"]==1])
no = len(d[d["Sentiment"]==1])
t = d[d["Sentiment"]==0][:no]
s = d[d["Sentiment"]==1]
df = pd.concat([s,t],ignore_index="True")
len(df)<jupyter_output><empty_output><jupyter_text>MODEL DEVELOPMENT AND EVALUATION  Splitting into Train and Test Data<jupyter_code>from sklearn.model_selection import train_test_split

x= df["Text"]
y = df["Sentiment"].astype("int")
x_train,x_test, y_train, y_test = train_test_split(x,y,test_size=0.2,random_state=42)
cv = CountVectorizer(ngram_range=(1,3))
tf = TfidfVectorizer(ngram_range=(1,3))
x1 = cv.fit_transform(x_train)
x2 = tf.fit_transform(x_train)
from sklearn import metrics

accuracy = {}
model = {}
vectorizer = {"CountVectorizer":cv,"TfidfVectorizer":tf}<jupyter_output><empty_output><jupyter_text>Support Vector Machine  CountVectorizer<jupyter_code>from sklearn import svm

classifier=svm.SVC()
classifier.fit(x1,y_train)
y_predict1=classifier.predict(cv.transform(x_test))
k = "Support Vector Machine with CountVectorizer"
model[k]=classifier
b1=metrics.accuracy_score(y_test, y_predict1)
recall = metrics.recall_score(y_test, y_predict1)
accuracy[k] = float("{0:.4f}".format(b1))
print(k)
print("Accuracy: ",accuracy[k])
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Support Vector Machine with CountVectorizer
Accuracy:  0.7813
Recall  : 0.6112<jupyter_text>TfidfVectorizer<jupyter_code>classifier=svm.SVC()
classifier.fit(x2,y_train)
y_predict2=classifier.predict(tf.transform(x_test))
k = "Support Vector Machine with TfidfVectorizer"
model[k]=classifier
b2=metrics.accuracy_score(y_test, y_predict2)
recall = metrics.recall_score(y_test, y_predict2)
accuracy[k] = float("{0:.4f}".format(b2))
print(k)
print("Accuracy: ",accuracy[k])
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Support Vector Machine with TfidfVectorizer
Accuracy:  0.8196
Recall  : 0.7365<jupyter_text>Multinomial Naive Bayes  Count Vectorizer<jupyter_code>from sklearn.naive_bayes import MultinomialNB

nb = MultinomialNB()
nb.fit(x1,y_train)
y_predict3=classifier.predict(cv.transform(x_test))
k = "Multinomial Naive Bayes with CountVectorizer"
model[k]=nb
b3=metrics.accuracy_score(y_test, y_predict3)
recall = metrics.recall_score(y_test, y_predict3)
accuracy[k] = float("{0:.4f}".format(b3))
print(k)
print("Accuracy: ",accuracy[k])
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Multinomial Naive Bayes with CountVectorizer
Accuracy:  0.5524
Recall  : 0.1469<jupyter_text>TfidfVectorizer<jupyter_code>nb = MultinomialNB()
nb.fit(x2,y_train)
y_predict4=classifier.predict(tf.transform(x_test))
k = "Multinomial Naive Bayes with TfidfVectorizer"
model[k]=nb
b4=metrics.accuracy_score(y_test,y_predict4)
recall = metrics.recall_score(y_test, y_predict4)
accuracy[k] = float("{0:.4f}".format(b4))
print(k)
print("Accuracy: ",accuracy[k])
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Multinomial Naive Bayes with TfidfVectorizer
Accuracy:  0.8196
Recall  : 0.7365<jupyter_text>Decision Tree Classifier  CountVectorizer<jupyter_code>from sklearn.tree import DecisionTreeClassifier

dtc = DecisionTreeClassifier()
dtc.fit(x1,y_train)
y_pred1 = dtc.predict(cv.transform(x_test))
k = "Decision Tree Classifier with CountVectorizer"
model[k]=dtc
a1 = metrics.accuracy_score(y_test, y_pred1)
recall = metrics.recall_score(y_test, y_pred1)
accuracy[k] = a1
print(k)
print("Accuracy: {0:.4f}".format(a1))
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Decision Tree Classifier with CountVectorizer
Accuracy: 0.9019
Recall  : 0.8985<jupyter_text>TfidfVectorizer<jupyter_code>dtc = DecisionTreeClassifier()
dtc.fit(x2,y_train)
y_pred2 = dtc.predict(tf.transform(x_test))
k = "Decision Tree Classifier with TfidfVectorizer"
model[k]=dtc
a2 = metrics.accuracy_score(y_test, y_pred2)
recall = metrics.recall_score(y_test, y_pred2)
accuracy[k] = a2
print(k)
print("Accuracy: {0:.4f}".format(a2))
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Decision Tree Classifier with TfidfVectorizer
Accuracy: 0.8918
Recall  : 0.8877<jupyter_text>Random Forest Classifier  CountVectorizer<jupyter_code>from sklearn.ensemble import RandomForestClassifier

rfc = RandomForestClassifier()
rfc.fit(x1,y_train)
y_pred3 = rfc.predict(cv.transform(x_test))
k = "Random Forest Classifier with CountVectorizer"
model[k]=rfc
a3 = metrics.accuracy_score(y_test, y_pred3)
recall = metrics.recall_score(y_test, y_pred3)
accuracy[k] = a3
print(k)
print("Accuracy: {0:.4f}".format(a3))
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Random Forest Classifier with CountVectorizer
Accuracy: 0.8433
Recall  : 0.7430<jupyter_text>TfidfVectorizer<jupyter_code>rfc = RandomForestClassifier()
rfc.fit(x2,y_train)
y_pred4 = rfc.predict(tf.transform(x_test))
k = "Random Forest Classifier with TfidfVectorizer"
model[k]=rfc
a4 = metrics.accuracy_score(y_test, y_pred4)
recall = metrics.recall_score(y_test, y_pred4)
accuracy[k] = a4
print(k)
print("Accuracy: {0:.4f}".format(a4))
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Random Forest Classifier with TfidfVectorizer
Accuracy: 0.8422
Recall  : 0.7603<jupyter_text>K Nearest Neighbors  CountVectorizer<jupyter_code>from sklearn.neighbors import KNeighborsClassifier

classifier = KNeighborsClassifier()
classifier.fit(x1,y_train)
Y_pred1 = classifier.predict(cv.transform(x_test))
k = "KNeighborsClassifier with CountVectorizer"
model[k]=classifier
c1 = metrics.accuracy_score(y_test, Y_pred1)
recall = metrics.recall_score(y_test, Y_pred1)
accuracy[k] = c1
print(k)
print("Accuracy: {0:.4f}".format(c1))
print("Recall  : {0:.4f}".format(recall))<jupyter_output>KNeighborsClassifier with CountVectorizer
Accuracy: 0.5829
Recall  : 0.2311<jupyter_text>TfidfVectorizer<jupyter_code>classifier = KNeighborsClassifier()
classifier.fit(x2,y_train)
Y_pred2 = classifier.predict(tf.transform(x_test))
k = "KNeighborsClassifier with TfidfVectorizer"
model[k]=classifier
c2 = metrics.accuracy_score(y_test, Y_pred2)
recall = metrics.recall_score(y_test, Y_pred2)
accuracy[k] = c2
print(k)
print("Accuracy: {0:.4f}".format(c2))
print("Recall  : {0:.4f}".format(recall))<jupyter_output>KNeighborsClassifier with TfidfVectorizer
Accuracy: 0.7294
Recall  : 0.7430<jupyter_text>Logistic Regression  CountVectorizer<jupyter_code>from sklearn.linear_model import LogisticRegression

lr = LogisticRegression()
lr.fit(x1,y_train)
Y_pred3 = lr.predict(cv.transform(x_test))
k = "Logistic Regression with CountVectorizer"
model[k]=lr
c3 = metrics.accuracy_score(y_test, Y_pred3)
recall = metrics.recall_score(y_test, Y_pred3)
accuracy[k] = c3
print(k)
print("Accuracy: {0:.4f}".format(c3))
print("Recall  : {0:.4f}".format(recall))<jupyter_output>Logistic Regression with CountVectorizer
Accuracy: 0.8286
Recall  : 0.7257<jupyter_text>TfidfVectorizer<jupyter_code>lr = LogisticRegression()
lr.fit(x2,y_train)
Y_pred4 = lr.predict(tf.transform(x_test))
k = "Logistic Regression with TfidfVectorizer"
model[k]=lr
c4 = metrics.accuracy_score(y_test, Y_pred4)
recall = metrics.recall_score(y_test, Y_pred4)
accuracy[k] = c4
print(k)
print("Accuracy: {0:.4f}".format(c4))
print("Recall  : {0:.4f}".format(recall))
sorted(accuracy)
ad = pd.DataFrame({"Accuracy":accuracy})
ad
m = max(accuracy,key=accuracy.get)
a = accuracy[m]
print("The model with highest accuracy is",m,"with an Accuracy of","{:.2f}".format(a*100),"%")
import pickle

if("CountVectorizer" in m):
    v = vectorizer["CountVectorizer"]
elif("TfidfVectorizer" in m):
    v = vectorizer["TfidfVectorizer"]

pickle.dump(v,open("vectorizer.pkl","wb"))
pickle.dump(model[m],open("model.pkl","wb"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1489417298144761, 'fraction_numerical': 0.025694625903666928, 'mean_word_length': 6.598941098610192, 'pattern_counts': {'":': 4, '<': 54, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '202811', 'license': 'MIT', 'n_tokens_mistral': 4752, 'n_tokens_neox': 4543, 'n_words': 719, 'path': 'Final Code.ipynb', 'repo_name': 'kumar306/Sentiment-Analysis-for-Depression-Detection', 'size': '11481'}