<jupyter_start><jupyter_text>MODELS<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from collections import Counter
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import train_test_split, StratifiedKFold
import pickle
from sklearn.metrics import (
    f1_score,
    accuracy_score,
    precision_score,
    recall_score,
    classification_report,
    confusion_matrix,
    plot_confusion_matrix,
    plot_precision_recall_curve,
    plot_roc_curve,
)
from sklearn.naive_bayes import MultinomialNB
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
import sklearn as sk
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from nltk.sentiment.vader import SentimentIntensityAnalyzer as VS
from sklearn.preprocessing import MinMaxScaler
import imblearn
from imblearn.under_sampling import RandomUnderSampler
from imblearn.over_sampling import RandomOverSampler
import re
import preprocessor as pproc
from cleantext import clean
import string
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize, sent_tokenize
import spacy
from yellowbrick.datasets import load_game
from yellowbrick.model_selection import LearningCurve
import dataframe_image as dfi
from sklearn.model_selection import cross_val_score
with open("serialized/data.pkl", "rb") as f:
    data = pickle.load(f)<jupyter_output><empty_output><jupyter_text>1. ORIGINAL DATASET<jupyter_code>tfidf_vectorizer = TfidfVectorizer()
X = tfidf_vectorizer.fit_transform(data['lemmatized'])
y = data["label"]
X.shape
Counter(y)
X_train_o, X_test_o, y_train_o, y_test_o = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
# Multinomial Naive Bayes
mnb_o = MultinomialNB(alpha=0.01)
mnb_o.fit(X_train_o, y_train_o)
y_mnb_o = mnb_o.predict(X_test_o)
as_mnb_o = accuracy_score(y_test_o, y_mnb_o)
f1s_mnb_o = f1_score(y_test_o, y_mnb_o, average= "weighted")
prec_mnb_o = precision_score(y_test_o, y_mnb_o)
rec_mnb_o = recall_score(y_test_o, y_mnb_o, average='weighted')
print("\nMultinomial Naive Bayes - Original dataset\n")
plot_confusion_matrix(mnb_o, X_test_o ,y_test_o, cmap= "Blues")
print(classification_report(y_test_o,y_mnb_o))
print(accuracy_score(y_test_o, y_mnb_o))
# Support Vector Machine
svc_o = SVC(verbose=10, random_state=42)
svc_o.fit(X_train_o, y_train_o)
y_svc_o = svc_o.predict(X_test_o)
as_svc_o = accuracy_score(y_test_o, y_svc_o)
f1s_svc_o = f1_score(y_test_o, y_svc_o, average="weighted")
prec_svc_o = precision_score(y_test_o, y_svc_o)
rec_svc_o = recall_score(y_test_o, y_svc_o, average='weighted')
print("\nSupport Vector Machine - Original dataset\n")
plot_confusion_matrix(svc_o, X_test_o ,y_test_o, cmap="Blues")
print(classification_report(y_test_o,y_svc_o))
print(accuracy_score(y_test_o, y_svc_o))
# Logistic
log_fit_o = LogisticRegression(random_state=42).fit(X_train_o, y_train_o)
y_log_o = log_fit_o.predict(X_test_o)
as_log_o = accuracy_score(y_test_o, y_log_o)
f1s_log_o = f1_score(y_test_o, y_log_o, average= "weighted")
prec_log_o = precision_score(y_test_o, y_log_o)
rec_log_o = recall_score(y_test_o, y_log_o, average='weighted')
print("\nLogistic Regression - Original dataset\n")
plot_confusion_matrix(log_fit_o, X_test_o ,y_test_o, cmap="Blues")
print(classification_report(y_test_o,y_log_o))
print(accuracy_score(y_test_o, y_log_o))
# Random Forest
rf_o = RandomForestClassifier(n_estimators=10, random_state=42)
rf_o.fit(X_train_o, y_train_o) 
y_rf_o = rf_o.predict(X_test_o)
as_rf_o = accuracy_score(y_test_o, y_rf_o)
f1s_rf_o = f1_score(y_test_o, y_rf_o, average= "weighted")
prec_rf_o = precision_score(y_test_o, y_rf_o)
rec_rf_o = recall_score(y_test_o, y_rf_o, average='weighted')
print("\nRandom Forest - Original dataset\n")
plot_confusion_matrix(rf_o, X_test_o,y_test_o, cmap="Blues")
print(classification_report(y_test_o,y_rf_o))
print(accuracy_score(y_test_o, y_rf_o))
# Decision Tree
tree_o = DecisionTreeClassifier(random_state=42)
tree_o.fit(X_train_o, y_train_o)
y_tree_o = tree_o.predict(X_test_o)
as_tree_o = accuracy_score(y_test_o, y_tree_o)
f1s_tree_o = f1_score(y_test_o, y_tree_o, average= "weighted")
prec_tree_o = precision_score(y_test_o, y_tree_o)
rec_tree_o = recall_score(y_test_o, y_tree_o, average='weighted')
print("\nDecision Tree - Original dataset\n")
plot_confusion_matrix(tree_o, X_test_o,y_test_o, cmap="Blues")
print(classification_report(y_test_o, y_tree_o))
print(accuracy_score(y_test_o, y_tree_o))
s1 = pd.Series([as_mnb_o, as_svc_o, as_log_o, as_rf_o, as_tree_o], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s2 = pd.Series([f1s_mnb_o, f1s_svc_o, f1s_log_o, f1s_rf_o, f1s_tree_o], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s3 = pd.Series([prec_mnb_o, prec_svc_o, prec_log_o, prec_rf_o, prec_tree_o], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s4 = pd.Series([rec_mnb_o, rec_svc_o, rec_log_o, rec_rf_o, rec_tree_o], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
d = {'Accuracy Score' : s1, 'Precision Score': s3, 'Recall Score' : s4, 'F1 Score' : s2 }
dfm = pd.DataFrame(d)
print("\nOriginal dataset\n")
dfm.style.highlight_max(color = 'lightgreen', axis = 0).highlight_min(color = 'tomato', axis = 0)
df_styled = dfm
dfi.export(df_styled,"images_d/or_data.jpg")
# Learning curve Original Dataset
cv = StratifiedKFold(n_splits=10)
sizes = np.linspace(0.1, 1, 20 )

model = LogisticRegression()
visualizer = LearningCurve(model, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=-1, shuffle=True)

vis = plt.figure()
visualizer.fit(X, y)
vis.savefig("images_d\Learning curve Original Dataset.png")
visualizer.show()<jupyter_output><empty_output><jupyter_text>2. BALANCED DATASET  2.1 Under Sample<jupyter_code>undersample = RandomUnderSampler(sampling_strategy='majority', random_state=42)
X_under, y_under = undersample.fit_resample(X, y)
print(Counter(y_under))
X_train_i, X_test_i, y_train_i, y_test_i = train_test_split(X_under, y_under, test_size=0.2, random_state=42, stratify=y_under)
# Multinomial Naive Bayes
mnb_i = MultinomialNB(alpha=0.01)
mnb_i.fit(X_train_i, y_train_i)
y_mnb_i = mnb_i.predict(X_test_i)
as_mnb_i = accuracy_score(y_test_i, y_mnb_i)
f1s_mnb_i = f1_score(y_test_i, y_mnb_i, average= "weighted")
prec_mnb_i = precision_score(y_test_i, y_mnb_i)
rec_mnb_i = recall_score(y_test_i, y_mnb_i, average='weighted')
print("\nMultinomoial Naive Bayes - Balanced Under dataset\n")
plot_confusion_matrix(mnb_i, X_test_i,y_test_i, cmap = "Blues")
print(classification_report(y_test_i, y_mnb_i))
print(accuracy_score(y_test_i, y_mnb_i))
# Support Vector Machine
svc_i = SVC(verbose=10, random_state=42)
svc_i.fit(X_train_i, y_train_i)
y_svc_i = svc_i.predict(X_test_i)
as_svc_i = accuracy_score(y_test_i, y_svc_i)
f1s_svc_i = f1_score(y_test_i, y_svc_i, average="weighted")
prec_svc_i = precision_score(y_test_i, y_svc_i)
rec_svc_i = recall_score(y_test_i, y_svc_i, average='weighted')
print("\nSupport Vector Machine - Balanced Under dataset\n")
plot_confusion_matrix(svc_i, X_test_i, y_test_i, cmap="Blues")
print(classification_report(y_test_i, y_svc_i))
print(accuracy_score(y_test_i, y_svc_i))
# Logistic
log_fit_i = LogisticRegression(random_state=42).fit(X_train_i, y_train_i)
y_log_i = log_fit_i.predict(X_test_i)
as_log_i = accuracy_score(y_test_i, y_log_i)
f1s_log_i = f1_score(y_test_i, y_log_i, average= "weighted")
prec_log_i = precision_score(y_test_i, y_log_i)
rec_log_i = recall_score(y_test_i, y_log_i, average='weighted')
print("\nLogistic Regression - Balanced Under dataset\n")
plot_confusion_matrix(log_fit_i, X_test_i, y_test_i, cmap="Blues")
print(classification_report(y_test_i, y_log_i))
print(accuracy_score(y_test_i, y_log_i))
# Random Forest
rf_i = RandomForestClassifier(n_estimators=10, random_state=42)
rf_i.fit(X_train_i, y_train_i) 
y_rf_i = rf_i.predict(X_test_i)
as_rf_i = accuracy_score(y_test_i, y_rf_i)
f1s_rf_i = f1_score(y_test_i, y_rf_i, average= "weighted")
prec_rf_i = precision_score(y_test_i, y_rf_i)
rec_rf_i = recall_score(y_test_i, y_rf_i, average='weighted')
print("\nRandom Forest - Balanced Under dataset\n")
plot_confusion_matrix(rf_i, X_test_i, y_test_i, cmap="Blues")
print(classification_report(y_test_i, y_rf_i))
print(accuracy_score(y_test_i, y_rf_i))
# Decision Tree
tree_i = DecisionTreeClassifier(random_state=42)
tree_i.fit(X_train_i, y_train_i)
y_tree_i = tree_i.predict(X_test_i)
as_tree_i = accuracy_score(y_test_i, y_tree_i)
f1s_tree_i = f1_score(y_test_i, y_tree_i, average= "weighted")
prec_tree_i = precision_score(y_test_i, y_tree_i)
rec_tree_i = recall_score(y_test_i, y_tree_i, average='weighted')
print("\nDecision Tree - Balanced Under dataset\n")
plot_confusion_matrix(tree_i, X_test_i, y_test_i, cmap="Blues")
print(classification_report(y_test_i, y_tree_i))
print(accuracy_score(y_test_i, y_tree_i))
s1_i = pd.Series([as_mnb_i, as_svc_i, as_log_i, as_rf_i, as_tree_i], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s2_i = pd.Series([f1s_mnb_i, f1s_svc_i, f1s_log_i, f1s_rf_i, f1s_tree_i], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s3_i = pd.Series([prec_mnb_i, prec_svc_i, prec_log_i, prec_rf_i, prec_tree_i], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s4_i = pd.Series([rec_mnb_i, rec_svc_i, rec_log_i, rec_rf_i, rec_tree_i], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
d_i = {'Accuracy Score' : s1_i, 'Precision Score': s3_i, 'Recall Score' : s4_i, 'F1 Score' : s2_i}
dfm_i = pd.DataFrame(d_i)
print("\nBalanced Under dataset\n")
dfm_i = dfm_i.style.highlight_max(color = 'lightgreen', axis = 0).highlight_min(color = 'tomato', axis = 0)
dfm_i
df_styled = dfm_i
dfi.export(df_styled,"images_d/under_data.jpg")
objects = ('Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic', 'Random Forest', 'Decision Tree')
y_pos = np.arange(len(objects))
performance_f1 = [f1s_mnb_i, f1s_svc_i, f1s_log_i, f1s_rf_i, f1s_tree_i]
clrs = ['lightblue' if (x < max(performance_f1)) else 'royalblue' for x in performance_f1 ]
plt.bar(y_pos, performance_f1, align='center', alpha=1, color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('F1-score')
plt.title('Model Comparision for F1-score')
plt.show()
performance_prec = [prec_mnb_i, prec_svc_i, prec_log_i, prec_rf_i, prec_tree_i]
clrs = ['lightblue' if (x < max(performance_prec)) else 'royalblue' for x in performance_prec ]
plt.bar(y_pos, performance_prec, align='center', alpha=1, color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('Precision')
plt.title('Model Comparision for Precision')
plt.show()
plot_precision_recall_curve(svc_i , X_test_i, y_test_i)
plot_roc_curve(svc_i,  X_test_i, y_test_i)
# Learning curve UnderSampled dataset
cv = StratifiedKFold(n_splits=10)
sizes = np.linspace(0.1, 1, 20 )

model = LogisticRegression()
visualizer = LearningCurve(model, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=-1, shuffle=True)

vis = plt.figure()
visualizer.fit(X_under, y_under)
vis.savefig("images_d\Learning curve UnderSampled dataset.png")
visualizer.show()
with open("serialized/vect.pkl", "wb") as f:
    pickle.dump(tfidf_vectorizer, f)

with open("serialized/log_i.pkl", "wb") as f:
    pickle.dump(svc_i, f)<jupyter_output><empty_output><jupyter_text>2.2. Over Sample<jupyter_code>oversample = RandomOverSampler(sampling_strategy='minority', random_state=42)
X_over, y_over = oversample.fit_resample(X, y)
print(Counter(y_over))
X_train_s, X_test_s, y_train_s, y_test_s = train_test_split(X_over, y_over, test_size=0.2, random_state=42, stratify=y_over)
# Multinomial Naive Bayes
mnb_s = MultinomialNB(alpha=0.01)
mnb_s.fit(X_train_s, y_train_s)
y_mnb_s = mnb_s.predict(X_test_s)
as_mnb_s = accuracy_score(y_test_s, y_mnb_s)
f1s_mnb_s = f1_score(y_test_s, y_mnb_s, average= "weighted")
prec_mnb_s = precision_score(y_test_s, y_mnb_s)
rec_mnb_s = recall_score(y_test_s, y_mnb_s, average='weighted')
print("\nMultinomoial Naive Bayes - Balanced Over dataset\n")
plot_confusion_matrix(mnb_s, X_test_s,y_test_s, cmap = "Blues")
print(classification_report(y_test_s, y_mnb_s))
print(accuracy_score(y_test_s, y_mnb_s))
# Support Vector Machine
svc_s = SVC(verbose=10, random_state=42)
svc_s.fit(X_train_s, y_train_s)
y_svc_s = svc_s.predict(X_test_s)
as_svc_s = accuracy_score(y_test_s, y_svc_s)
f1s_svc_s = f1_score(y_test_s, y_svc_s, average="weighted")
prec_svc_s = precision_score(y_test_s, y_svc_s)
rec_svc_s = recall_score(y_test_s, y_svc_s, average='weighted')
print("\nSupport Vector Machine - Balanced Over dataset\n")
plot_confusion_matrix(svc_s, X_test_s, y_test_s, cmap="Blues")
print(classification_report(y_test_s, y_svc_s))
print(accuracy_score(y_test_s, y_svc_s))
# Logistic
log_fit_s = LogisticRegression(random_state=42).fit(X_train_s, y_train_s)
y_log_s = log_fit_s.predict(X_test_s)
as_log_s = accuracy_score(y_test_s, y_log_s)
f1s_log_s = f1_score(y_test_s, y_log_s, average= "weighted")
prec_log_s = precision_score(y_test_s, y_log_s)
rec_log_s = recall_score(y_test_s, y_log_s, average='weighted')
print("\nLogistic Regression - Balanced Over dataset\n")
plot_confusion_matrix(log_fit_s, X_test_s, y_test_s, cmap="Blues")
print(classification_report(y_test_s, y_log_s))
print(accuracy_score(y_test_s, y_log_s))
# Random Forest
rf_s = RandomForestClassifier(n_estimators=10, random_state=42)
rf_s.fit(X_train_s, y_train_s) 
y_rf_s = rf_s.predict(X_test_s)
as_rf_s = accuracy_score(y_test_s, y_rf_s)
f1s_rf_s = f1_score(y_test_s, y_rf_s, average= "weighted")
prec_rf_s = precision_score(y_test_s, y_rf_s)
rec_rf_s = recall_score(y_test_s, y_rf_s, average='weighted')
print("\nRandom Forest - Balanced Over dataset\n")
plot_confusion_matrix(rf_s, X_test_s, y_test_s, cmap="Blues")
print(classification_report(y_test_s, y_rf_s))
print(accuracy_score(y_test_s, y_rf_s))
# Decision Tree
tree_s = DecisionTreeClassifier(random_state=42)
tree_s.fit(X_train_s, y_train_s)
y_tree_s = tree_s.predict(X_test_s)
as_tree_s = accuracy_score(y_test_s, y_tree_s)
f1s_tree_s = f1_score(y_test_s, y_tree_s, average= "weighted")
prec_tree_s = precision_score(y_test_s, y_tree_s)
rec_tree_s = recall_score(y_test_s, y_tree_s, average='weighted')
print("\nDecision Tree - Balanced Over dataset\n")
plot_confusion_matrix(tree_s, X_test_s, y_test_s, cmap="Blues")
print(classification_report(y_test_s, y_tree_s))
print(accuracy_score(y_test_s, y_tree_s))
s1_s = pd.Series([as_mnb_s, as_svc_s, as_log_s, as_rf_s, as_tree_s], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s2_s = pd.Series([f1s_mnb_s, f1s_svc_s, f1s_log_s, f1s_rf_s, f1s_tree_s], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s3_s = pd.Series([prec_mnb_s, prec_svc_s, prec_log_s, prec_rf_s, prec_tree_s], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s4_s = pd.Series([rec_mnb_s, rec_svc_s, rec_log_s, rec_rf_s, rec_tree_s], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
d_s = {'Accuracy Score' : s1_s, 'Precision Score': s3_s, 'Recall Score' : s4_s, 'F1 Score' : s2_s}
dfm_s = pd.DataFrame(d_s)
print("\nBalanced Over dataset\n")
dfm_s = dfm_s.style.highlight_max(color = 'lightgreen', axis = 0).highlight_min(color = 'tomato', axis = 0)
dfm_s
df_styled = dfm_s
dfi.export(df_styled,"images_d/over_data.jpg")
objects = ('Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic', 'Random Forest', 'Decision Tree')
y_pos = np.arange(len(objects))
performance_f1 = [f1s_mnb_s, f1s_svc_s, f1s_log_s, f1s_rf_s, f1s_tree_s]
clrs = ['lightblue' if (x < max(performance_f1)) else 'royalblue' for x in performance_f1 ]
plt.bar(y_pos, performance_f1, align='center', alpha=1, color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('F1-score')
plt.title('Model Comparision for F1-score')
plt.show()
performance_prec = [prec_mnb_s, prec_svc_s, prec_log_s, prec_rf_s, prec_tree_s]
clrs = ['lightblue' if (x < max(performance_prec)) else 'royalblue' for x in performance_prec ]
plt.bar(y_pos, performance_prec, align='center', alpha=1, color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('Precision')
plt.title('Model Comparision for Precision')
plt.show()
plot_precision_recall_curve(rf_s, X_test_s, y_test_s)
plot_roc_curve(rf_s,  X_test_s, y_test_s)
# Learning curve  OverSampled dataset
cv = StratifiedKFold(n_splits=10)
sizes = np.linspace(0.1, 1, 20)

model = LogisticRegression()
visualizer = LearningCurve(model, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=-1, shuffle=True)

visualizer.fit(X_over, y_over)
visualizer.show()<jupyter_output><empty_output><jupyter_text>3 MODELS ON POS  3.1 Nouns + Verbs + Adjectives + Pronouns + Proper Nouns<jupyter_code>def filter_text_pos(x):
    final_pos_text = []
    for elem in x:
        for pos in pos_list:
            if elem[1] == pos:
                final_pos_text.append(elem[0])
    
    return " ".join(final_pos_text)
pos_list = ["NOUN" ,  "VERB" ,  "ADJ",  "PRON", "PROPN"]
data["NOUN_VERB_ADJ_PRON_PROPN"] = data.apply(lambda x: filter_text_pos(x["POS_spacy"]), axis=1)
tfidf_vectorizer_pos1 = TfidfVectorizer()
X_P = tfidf_vectorizer_pos1.fit_transform(data["NOUN_VERB_ADJ_PRON_PROPN"])
y_P = data["label"]
X_under_p, y_under_p = undersample.fit_resample(X_P, y_P)
X_train_p, X_test_p, y_train_p, y_test_p = train_test_split(X_under_p, y_under_p, test_size=0.2, random_state=42, stratify=y_under_p)
# Multinomial Naive Bayes
mnb_p = MultinomialNB(alpha=0.01)
mnb_p.fit(X_train_p, y_train_p)
y_mnb_p = mnb_p.predict(X_test_p)
as_mnb_p = accuracy_score(y_test_p, y_mnb_p)
f1s_mnb_p = f1_score(y_test_p, y_mnb_p, average= "weighted")
prec_mnb_p = precision_score(y_test_p, y_mnb_p)
rec_mnb_p = recall_score(y_test_p, y_mnb_p, average='weighted')
print("\nMultinomoial Naive Bayes - NOUN_VERB_ADJ_PRON_PROPN\n")
plot_confusion_matrix(mnb_p, X_test_p, y_test_p, cmap="Blues")
print(classification_report(y_test_p, y_mnb_p))
print(accuracy_score(y_test_p, y_mnb_p))
# Support Vector Machine
svc_p = SVC(verbose=10, random_state=42)
svc_p.fit(X_train_p, y_train_p)
y_svc_p = svc_p.predict(X_test_p)
as_svc_p = accuracy_score(y_test_p, y_svc_p)
f1s_svc_p = f1_score(y_test_p, y_svc_p, average="weighted")
prec_svc_p = precision_score(y_test_p, y_svc_p)
rec_svc_p = recall_score(y_test_p, y_svc_p, average='weighted')
print("\nSupport Vector Machine - NOUN_VERB_ADJ_PRON_PROPN\n")
plot_confusion_matrix(svc_p, X_test_p, y_test_p, cmap="Blues")
print(classification_report(y_test_p, y_svc_p))
print(accuracy_score(y_test_p, y_svc_p))
# Logistic
log_fit_p = LogisticRegression(random_state=42).fit(X_train_p, y_train_p)
y_log_p = log_fit_p.predict(X_test_p)
as_log_p = accuracy_score(y_test_p, y_log_p)
f1s_log_p = f1_score(y_test_p, y_log_p, average= "weighted")
prec_log_p = precision_score(y_test_p, y_log_p)
rec_log_p = recall_score(y_test_p, y_log_p, average='weighted')
print("\nLogistic Regression - NOUN_VERB_ADJ_PRON_PROPN\n")
plot_confusion_matrix(log_fit_p, X_test_p, y_test_p, cmap="Blues")
print(classification_report(y_test_p, y_log_p))
print(accuracy_score(y_test_p, y_log_p))
# Random Forest
rf_p = RandomForestClassifier(n_estimators=10, random_state=42)
rf_p.fit(X_train_p, y_train_p) 
y_rf_p = rf_p.predict(X_test_p)
as_rf_p = accuracy_score(y_test_p, y_rf_p)
f1s_rf_p = f1_score(y_test_p, y_rf_p, average= "weighted")
prec_rf_p = precision_score(y_test_p, y_rf_p)
rec_rf_p = recall_score(y_test_p, y_rf_p, average='weighted')
print("\nRandom Forest - NOUN_VERB_ADJ_PRON_PROPN\n")
plot_confusion_matrix(rf_p, X_test_p, y_test_p, cmap="Blues")
print(classification_report(y_test_p, y_rf_p))
print(accuracy_score(y_test_p, y_rf_p))
# Decision Tree
tree_p = DecisionTreeClassifier(random_state=42)
tree_p.fit(X_train_p, y_train_p)
y_tree_p = tree_p.predict(X_test_p)
as_tree_p = accuracy_score(y_test_p, y_tree_p)
f1s_tree_p = f1_score(y_test_p, y_tree_p, average= "weighted")
prec_tree_p = precision_score(y_test_p, y_tree_p)
rec_tree_p = recall_score(y_test_p, y_tree_p, average='weighted')
print("\nDecision Tree - NOUN_VERB_ADJ_PRON_PROPN\n")
plot_confusion_matrix(tree_p, X_test_p, y_test_p, cmap="Blues")
print(classification_report(y_test_p, y_tree_p))
print(accuracy_score(y_test_p, y_tree_p))
s1_p = pd.Series([as_mnb_p, as_svc_p, as_log_p, as_rf_p, as_tree_p], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s2_p = pd.Series([f1s_mnb_p, f1s_svc_p, f1s_log_p, f1s_rf_p,  f1s_tree_p], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s3_p = pd.Series([prec_mnb_p, prec_svc_p, prec_log_p, prec_rf_p, prec_tree_p], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s4_p = pd.Series([rec_mnb_p, rec_svc_p, rec_log_p, rec_rf_p, rec_tree_p], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
d_p = {'Accuracy Score' : s1_p, 'Precision Score': s3_p, 'Recall Score' : s4_p, 'F1 Score' : s2_p}
dfm_p = pd.DataFrame(d_p)
print("\nNOUN_VERB_ADJ_PRON_PROPN\n")
dfm_p = dfm_p.style.highlight_max(color = 'lightgreen', axis = 0).highlight_min(color = 'tomato', axis = 0)
dfm_p
df_styled = dfm_p
dfi.export(df_styled,"images_d/pos1_data.jpg")
objects = ('Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic', 'Random Forest', 'Decision Tree')
y_pos = np.arange(len(objects))
performance_f1 = [f1s_mnb_p, f1s_svc_p, f1s_log_p, f1s_rf_p, f1s_tree_p]
clrs = ['lightblue' if (x < max(performance_f1)) else 'royalblue' for x in performance_f1 ]
plt.bar(y_pos, performance_f1, align='center',  color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('F1-score')
plt.title('Model Comparision for F1-score')
plt.show()
performance_prec = [prec_mnb_p, prec_svc_p, prec_log_p, prec_rf_p, prec_tree_p]
clrs = ['lightblue' if (x < max(performance_prec)) else 'royalblue' for x in performance_prec ]
plt.bar(y_pos, performance_prec, align='center', alpha=1, color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('Precision')
plt.title('Model Comparision for Precision')
plt.show()
plot_precision_recall_curve(log_fit_p, X_test_p, y_test_p)
plot_roc_curve(log_fit_p,  X_test_p, y_test_p)
# Learning curve POS 1 dataset
cv = StratifiedKFold(n_splits=10)
sizes = np.linspace(0.1, 1, 50 )

model = LogisticRegression()
visualizer = LearningCurve(model, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=-1, shuffle=True)

vis = plt.figure()
visualizer.fit(X_under_p, y_under_p)
vis.savefig("images_d\Learning curve POS 1 dataset.png")
visualizer.show()
with open("serialized/vect_pos1.pkl", "wb") as f:
    pickle.dump(tfidf_vectorizer_pos1, f)

with open("serialized/log_pos1.pkl", "wb") as f:
    pickle.dump(log_fit_p, f)<jupyter_output><empty_output><jupyter_text>3.2 Nouns + Proper Nouns + Verbs + Adjectives + Pronouns + Subordinating Conjunction + Adpositions +  Coordinating Conjunctions + Other<jupyter_code>tfidf_vectorizer_pos2 = TfidfVectorizer()
pos_list = ["NOUN","PROPN", "VERB", "ADJ", "PRON", "SCONJ", "ADP", "CCONJ", "X"] #NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X
data["NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X"] = data.apply(lambda x: filter_text_pos(x["POS_spacy"]), axis=1)
X_N = tfidf_vectorizer_pos2.fit_transform(data["NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X"])
y_N = data["label"]
X_under_n, y_under_n = undersample.fit_resample(X_N, y_N)
X_train_n, X_test_n, y_train_n, y_test_n = train_test_split(X_under_n, y_under_n, test_size=0.2, random_state=42, stratify=y_under_n)
# Multinomial Naive Bayes
mnb_n = MultinomialNB(alpha=0.01)
mnb_n.fit(X_train_n, y_train_n)
y_mnb_n = mnb_n.predict(X_test_n)
as_mnb_n = accuracy_score(y_test_n, y_mnb_n)
f1s_mnb_n = f1_score(y_test_n, y_mnb_n, average= "weighted")
prec_mnb_n = precision_score(y_test_n, y_mnb_n)
rec_mnb_n = recall_score(y_test_n, y_mnb_n, average='weighted')
print("\nMultinomoial Naive Bayes - NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X\n")
plot_confusion_matrix(mnb_n, X_test_n, y_test_n, cmap="Blues")
print(classification_report(y_test_n, y_mnb_n))
print(accuracy_score(y_test_n, y_mnb_n))
# Support Vector Machine
svc_n = SVC(verbose=10, random_state=42)
svc_n.fit(X_train_n, y_train_n)
y_svc_n = svc_n.predict(X_test_n)
as_svc_n = accuracy_score(y_test_n, y_svc_n)
f1s_svc_n = f1_score(y_test_n, y_svc_n, average="weighted")
prec_svc_n = precision_score(y_test_n, y_svc_n)
rec_svc_n = recall_score(y_test_n, y_svc_n, average='weighted')
print("\nSupport Vector Machine - NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X\n")
plot_confusion_matrix(svc_n, X_test_n, y_test_n, cmap="Blues")
print(classification_report(y_test_n, y_svc_n))
print(accuracy_score(y_test_n, y_svc_n))
# Logistic
log_fit_n = LogisticRegression(random_state=42).fit(X_train_n, y_train_n)
y_log_n = log_fit_n.predict(X_test_n)
as_log_n = accuracy_score(y_test_n, y_log_n)
f1s_log_n = f1_score(y_test_n, y_log_n, average= "weighted")
prec_log_n = precision_score(y_test_n, y_log_n)
rec_log_n = recall_score(y_test_n, y_log_n, average='weighted')
print("\nLogistic Regression - NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X\n")
plot_confusion_matrix(log_fit_n, X_test_n, y_test_n, cmap="Blues")
print(classification_report(y_test_n, y_log_n))
print(accuracy_score(y_test_n, y_log_n))
# Random Forest
rf_n = RandomForestClassifier(n_estimators=10, random_state=42)
rf_n.fit(X_train_n, y_train_n) 
y_rf_n = rf_n.predict(X_test_n)
as_rf_n = accuracy_score(y_test_n, y_rf_n)
f1s_rf_n = f1_score(y_test_n, y_rf_n, average= "weighted")
prec_rf_n = precision_score(y_test_n, y_rf_n)
rec_rf_n = recall_score(y_test_n, y_rf_n, average='weighted')
print("\nRandom Forest - NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X\n")
plot_confusion_matrix(rf_n, X_test_n, y_test_n, cmap="Blues")
print(classification_report(y_test_n, y_rf_n))
print(accuracy_score(y_test_n, y_rf_n))
# Decision Tree
tree_n = DecisionTreeClassifier(random_state=42)
tree_n.fit(X_train_n, y_train_n)
y_tree_n = tree_n.predict(X_test_n)
as_tree_n = accuracy_score(y_test_n, y_tree_n)
f1s_tree_n = f1_score(y_test_n, y_tree_n, average= "weighted")
prec_tree_n = precision_score(y_test_n, y_tree_n)
rec_tree_n = recall_score(y_test_n, y_tree_n, average='weighted')
print("\nDecision Tree - NOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X\n")
plot_confusion_matrix(tree_n, X_test_n, y_test_n, cmap="Blues")
print(classification_report(y_test_n, y_tree_n))
print(accuracy_score(y_test_n, y_tree_n))
s1_n = pd.Series([as_mnb_n, as_svc_n, as_log_n, as_rf_n, as_tree_n], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s2_n = pd.Series([f1s_mnb_n, f1s_svc_n, f1s_log_n, f1s_rf_n, f1s_tree_n], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s3_n = pd.Series([prec_mnb_n, prec_svc_n, prec_log_n, prec_rf_n, prec_tree_n], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
s4_n = pd.Series([rec_mnb_n, rec_svc_n, rec_log_n, rec_rf_n, rec_tree_n,], index=['Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic Regression', 'Random Forest', 'Decision Tree'])
d_n = {'Accuracy Score' : s1_n, 'Precision Score': s3_n, 'Recall Score' : s4_n, 'F1 Score' : s2_n}
dfm_n = pd.DataFrame(d_n)
print("\nNOUN_PROPN_VERB_ADJ_PRON_SCONJ_ADP_CCONJ_X\n")
dfm_n = dfm_n.style.highlight_max(color = 'lightgreen', axis = 0).highlight_min(color = 'tomato', axis = 0)
dfm_n
df_styled = dfm_n
dfi.export(df_styled,"images_d/pos2_data.jpg")
objects = ('Multinomial Naive Bayes', 'Support Vector Machine', 'Logistic', 'Random Forest', 'Decision Tree')
y_pos = np.arange(len(objects))
performance = [f1s_mnb_n, f1s_svc_n, f1s_log_n, f1s_rf_n, f1s_tree_n]
clrs = ['lightblue' if (x < max(performance)) else 'royalblue' for x in performance ]
plt.bar(y_pos, performance, align='center', alpha=1, color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('F1-score')
plt.title('Model Comparision for F1-score')
plt.show()
performance_prec = [prec_mnb_n, prec_svc_n, prec_log_n, prec_rf_n, prec_tree_n]
clrs = ['lightblue' if (x < max(performance_prec)) else 'royalblue' for x in performance_prec ]
plt.bar(y_pos, performance_prec, align='center', alpha=1, color=clrs)
plt.xticks(y_pos, objects, rotation=270)
plt.ylabel('Precision')
plt.title('Model Comparision for Precision')
plt.show()
plot_precision_recall_curve(log_fit_n, X_test_n, y_test_n)
plot_roc_curve(log_fit_n,  X_test_n, y_test_n)
# Learning curve POS 2 dataset
cv = StratifiedKFold(n_splits=10)
sizes = np.linspace(0.1, 1, 50)

model = LogisticRegression()
visualizer = LearningCurve(model, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=-1, shuffle=True)

vis = plt.figure()
visualizer.fit(X_under_n, y_under_n)
vis.savefig("images_d\Learning curve POS 2 dataset.png")
visualizer.show()
with open("serialized/vect_pos.pkl", "wb") as f:
    pickle.dump(tfidf_vectorizer_pos2, f)

with open("serialized/svc_pos2.pkl", "wb") as f:
    pickle.dump(log_fit_n, f)<jupyter_output><empty_output><jupyter_text>5. HATE SPEECH CLASSIFICATION<jupyter_code>def expand_contractions(text):
    cList = {
        "n't": " not",
        "/TD": " ",
        " PM ": " personal message ",
        " pm ": " personal message ",
        "PM ": "personal message ",
        " Donot ": " do not ",
        " MB ": " megabytes ",
        "I'm" : "I am",
        " 've " : " have ",
        " 're " : " are ",
        " 'll " : " will "
    }
    
    c_re = re.compile("(%s)" % "|".join(cList.keys()))

    return c_re.sub(lambda match: cList[match.group(0)], text)
def full_text_clean(text, nlp, filter_pos):
    aa = expand_contractions(text)
    
    bb = pproc.clean(
        clean(pproc.clean(aa),
              fix_unicode=True,              
              to_ascii=True,                  
              lower=True,                     
              no_line_breaks=True,           
              no_urls=True,                  
              no_emails=True,                
              no_phone_numbers=False,         
              no_numbers=False,               
              no_digits=False,                
              no_currency_symbols=False,      
              no_punct=True,                 
              replace_with_url=" ",
              replace_with_email=" ",
        )
    )
    
    swords = string.punctuation
    
    cc = (
        bb.lower()
        .replace(r"(@[a-z0-9]+)\w+", " ")
        .replace(r"www\S+", " ")
        .replace(r"com/watch", " ")
        .replace(r"\S*[.,:;!?-]\S*[^\s\.,:;!?-]", " ")
        .replace(r" th ", " ")
        .replace(r"\w*\d\w*", " ")
        .replace(r"rlm", " ")
        .replace(r"pttm", " ")
        .replace(r"ghlight", " ")
        .replace(r"[0-9]+(?:st| st|nd| nd|rd| rd|th| th)", " ")
        .replace(r"([^a-z \t])", " ")
        .replace(r" +", " ")
        .replace(r"http", " ")
        )
    
    cc = " ".join([i for i in cc.split() if not i in swords])
    
    doc = nlp(cc)
    if filter_pos:
        return filter_text_pos(doc)
    else:
        return " ".join([y.lemma_ for y in doc if len(y) > 1])
def filter_text_pos(x):
    final_pos_text = []
    for elem in x:
        for pos in ["NOUN" ,  "VERB" ,  "ADJ",  "PRON", "PROPN"]:
            if elem.pos_ == pos:
                final_pos_text.append(elem.text)

    return " ".join(final_pos_text)
def hate_predict(X, vect, clf, nlp, filter_pos):
    lista_pulita = [full_text_clean(text, nlp, filter_pos) for text in X]
    X_new = vect.transform(lista_pulita)
    classification = clf.predict(X_new)
    return classification
nlp = spacy.load("en_core_web_sm")
written_1 = "I hate all of you"
written_2 = "God hates blacks"
written_3 = "They are sick violent merciless animals."
under_s_1 = hate_predict([written_1], tfidf_vectorizer, svc_i, nlp, filter_pos=False)
under_s_2 = hate_predict([written_2], tfidf_vectorizer, svc_i, nlp, filter_pos=False)
under_s_3 = hate_predict([written_3], tfidf_vectorizer, svc_i, nlp, filter_pos=False)
under_s_1, under_s_2, under_s_3
pos_1 = hate_predict([written_1], tfidf_vectorizer_pos1, log_fit_p, nlp, filter_pos=True)
pos_2 = hate_predict([written_2], tfidf_vectorizer_pos1, log_fit_p, nlp, filter_pos=True)
pos_3 = hate_predict([written_3], tfidf_vectorizer_pos1, log_fit_p, nlp, filter_pos=True)
pos_1, pos_2, pos_3<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12489492013930588, 'fraction_numerical': 0.012219286657859974, 'mean_word_length': 6.923168411037108, 'pattern_counts': {'":': 7, '<': 37, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '217124', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 14515, 'n_tokens_neox': 14119, 'n_words': 2413, 'path': '03. Models.ipynb', 'repo_name': 'martinaviggiano/textsent_project', 'size': '33308'}