<jupyter_start><jupyter_text>Model Evaluation - Cross Validation Create text classification models using logistic regression & Naive Bayes and evaluate the performance of those models through Cross Validation and Parameter Tuning to select the best model.  Data set Download the data set and place it in the appropriate directory. The zip file of data set is available on university site - https://content.bellevue.edu/cst/dsc/550/id/data.zip.  The JSONL data files for this project are in reddit directory. Place the "controversial-comments.jsonl" and "categorized-comments.jsonl" in the directory.  Import libraries for dataset preparation, feature engineering, model training<jupyter_code>import pandas as pd
import string, numpy as np
import re

from sklearn import model_selection
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import MultinomialNB
from  sklearn.linear_model import LogisticRegression
from sklearn import metrics

from nltk.stem import WordNetLemmatizer
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.preprocessing import LabelEncoder

import nltk
from nltk.corpus import wordnet
from nltk.stem.snowball import SnowballStemmer
from nltk.stem import WordNetLemmatizer 
from nltk.stem.snowball import FrenchStemmer

# for measuring accuracy, precision, recall, f1 and auc scores
from sklearn.metrics import accuracy_score,precision_score, recall_score, f1_score, roc_auc_score
from sklearn.model_selection import cross_val_score
# load the dataset
df = pd.read_json('controversial-comments.jsonl', lines = 'True')
print(df.head())
print(len(df))
print("count of 0 contro :", len(df[df.con == 0]))
print("count of 1 contro :", len(df[df.con == 1]))
#Cleaning text
def textcleaning(text):
    
    text = text.lower()
    #removing \n
    text = re.sub(r'\n', '', text)
    text = re.sub(r'@\w+', '', text)
    # removing urls
    text = re.sub(r'http.?://[^\s]+[\s]?', ' ', text)
    # removing symbols and numbers
    text = re.sub('[^a-zA-Z\s]', '', text)
    # removing 3 letter words
    text = re.sub(r'(\b\w{1,3}\b)', '', text) 
    
    return text
#Applying text cleaning on text field to clean it up
df['clndtxt'] = df['txt'].apply(textcleaning)
#from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
def lemmatize_text(text):
    tokens = text.split()
    
    return [lemmatizer.lemmatize(w) for w in tokens]
df['lemmedtxt'] = df['clndtxt'].apply(lemmatize_text).apply(lambda x : " ".join(x))
df.head()
# Replace the column to latest transformed values
df['clndtxt'] = df['lemmedtxt']
df.head()
text, target = df['clndtxt'], df['con']
#split data into train and test
text_train, text_test, label_train, label_test = model_selection.train_test_split(df['clndtxt'], df['con'], 
                                                                                  test_size = 0.25)

# Checking lenghts of test and test
print("Number of observations in train", len(text_train))

print("Number of observations in train", len(text_test))

# create a count vectorizer object 
count_vect = CountVectorizer(analyzer='word', stop_words='english', lowercase=True, min_df = 2)

cvtrnsftexttrain = count_vect.fit_transform(text_train)
print(len(count_vect.get_feature_names())) 
print(count_vect.fit_transform(text_train).shape)
# converting testing text into matrix form for computation using transform
cvtrnsftexttest = count_vect.transform(text_test)
print(count_vect.transform(text_test).shape) 
# Confirming shapes of test and train transformations
print(cvtrnsftexttrain.shape)
print(cvtrnsftexttest.shape)<jupyter_output>(712500, 79527)
(237500, 79527)<jupyter_text>Logistic Regression Logistict Regression with penalty = L1<jupyter_code># Creating instance for logistic regression with penalty L1
logregmodel = LogisticRegression(n_jobs=-1, penalty='l1', random_state=0) 


# Applying logistic regression to train data set
logregmodel.fit(cvtrnsftexttrain, label_train)
# predict the labels on trained dataset
logregmodelpredtr = logregmodel.predict(cvtrnsftexttrain)
# predict the labels on test dataset
logregmodelpredtest = logregmodel.predict(cvtrnsftexttest)
# Testing total accurate predictions
print (np.sum(logregmodelpredtest == label_test)) 
print (np.sum(logregmodelpredtest == label_test)/237500) 
#Generate labelled performance metrics
print(metrics.classification_report(label_test, logregmodelpredtest)) 
accuracy_train = metrics.accuracy_score(label_train, logregmodelpredtr)
accuracy_test = metrics.accuracy_score(label_test, logregmodelpredtest)
print(accuracy_train, accuracy_test) 
# create rows list for data manipulation
rows = []

def fn_metrics(actual_label, predicted_label):
    """
    function that takes acutal labels and predicted labels and returns
    accuracy, auc, precision, recall and f1 scores
    average = 'micro' - to return global metrics
    """
    accuracy = accuracy_score(actual_label, predicted_label)
    auc =  roc_auc_score(actual_label, predicted_label)
    precision = precision_score(actual_label, predicted_label, average = 'micro')
    recall = recall_score(actual_label, predicted_label,  average = 'micro')
    f1 = f1_score(actual_label, predicted_label, average = 'micro')
    
    return (accuracy, auc, precision, recall, f1)
#print(fn_metrics(label_test, logregmodelpredtest))

acc, auc, prec, recall, f1 = fn_metrics(label_test, logregmodelpredtest)

l1row= ['Logistic Regression (L1)', acc, auc, prec, recall, f1]
print(l1row)

rows.append(l1row)
                    
# ['Logistic Regression (L1)', 0.9548042105263158, 0.5015922168040325, 0.1541095890410959, 0.0042726927459172045, 0.008314855875831485]<jupyter_output>['Logistic Regression (L1)', 0.9548042105263158, 0.5015922168040325, 0.9548042105263158, 0.9548042105263158, 0.9548042105263158]<jupyter_text>Logistic Regression for Penalty = L2<jupyter_code>logregmodel2 = LogisticRegression(n_jobs=-1, penalty='l2', random_state=0, solver = 'sag')
logregmodel2.fit(cvtrnsftexttrain, label_train)
# predict the labels on trained dataset
logregmodelpredtr2 = logregmodel2.predict(cvtrnsftexttrain)

# predict the labels on test dataset
logregmodelpredtest2 = logregmodel2.predict(cvtrnsftexttest)

print(metrics.classification_report(label_test, logregmodelpredtest2)) #, target_names=docs_to_train.target_names))
accuracy_trainl2 = metrics.accuracy_score(label_train, logregmodelpredtr2)
accuracy_testl2 = metrics.accuracy_score(label_test, logregmodelpredtest2)
print(accuracy_trainl2, accuracy_testl2)
#print(fn_metrics(label_test, logregmodelpredtest))

acc, auc, prec, recall, f1 = fn_metrics(label_test, logregmodelpredtest2)

l2row= ['Logistic Regression (L2)', acc, auc, prec, recall, f1]
print(l2row)

rows.append(l2row)
# ['Logistic Regression (L2)', 0.9556, 0.5002429900623745, 0.24, 0.000569692366122294, 0.0011366865586814436]<jupyter_output>['Logistic Regression (L2)', 0.9556, 0.5002429900623745, 0.9556, 0.9556, 0.9556]<jupyter_text>Using Naive Bayes Algorithm<jupyter_code>#from sklearn.naive_bayes import MultinomialNB

Nb_cv = MultinomialNB()
Nb_cv.fit(cvtrnsftexttrain, label_train)

# predict the labels on trained dataset
NB_cv_predtr = Nb_cv.predict(cvtrnsftexttrain)

# predict the labels on test dataset
NB_cv_predtst = Nb_cv.predict(cvtrnsftexttest)

# Calculating Accuracy scores
Acc_NB_cv_predtr = metrics.accuracy_score(label_train, NB_cv_predtr)
Acc_NB_cv_predtst = metrics.accuracy_score(label_test, NB_cv_predtst)
print(Acc_NB_cv_predtr, Acc_NB_cv_predtst) 

# Testing total accurate predictions for test
print (np.sum(NB_cv_predtst == label_test)) 
#print(fn_metrics(label_test, logregmodelpredtest))

acc, auc, prec, recall, f1 = fn_metrics(label_test, NB_cv_predtst)

nbrow= ['Naive Bayes', acc, auc, prec, recall, f1]
print(nbrow)

rows.append(nbrow)
# ['Naive Bayes', 0.9420547368421053, 0.5124417086564873, 0.10552027357107963, 0.041017850360805165, 0.05907288390537399]<jupyter_output>['Naive Bayes', 0.9420547368421053, 0.5124417086564873, 0.9420547368421053, 0.9420547368421053, 0.9420547368421053]<jupyter_text>Creating data frame for reporting<jupyter_code># create data frame to hold results
modelperf_df = pd.DataFrame()
modelperf_df = pd.DataFrame(rows, columns = ['Model', 'Accuracy', 'AUC', 'Precision', 'Recall', 'F1'])
modelperf_df<jupyter_output><empty_output><jupyter_text>Observations: Logisitc Regression penalty L2 has slightly better accuracy than other models  b. Cross Validation<jupyter_code>from sklearn.model_selection import KFold, cross_val_score

# Create k-Fold cross-validation 
kf3 = KFold(n_splits=3 # 3 fold cross validation
            , shuffle=True # to shuffle observations
            ,random_state=1)

#features = cvtrnsftexttrain #cvtrnsftexttest
#target = label_train #label_test

text, target = df['clndtxt'], df['con']

# create a count vectorizer object 
count_vect = CountVectorizer(analyzer='word', stop_words='english', lowercase=True, min_df = 2)

# Converting features into vectorized matrix for applying model
features = count_vect.fit_transform(text)
print(len(count_vect.get_feature_names())) 

# Conduct k-fold cross-validation
cv_results = cross_val_score(logregmodel2, # Logistic Regression with penalty L2
                            features, # Feature matrix
                            target, # Target vector
                            cv=kf3, # Cross-validation technique
                            scoring="roc_auc", # Loss function
                            n_jobs=-1) # Use all CPU scores
print(cv_results)
# for test [0.5289055  0.53574204 0.60549728]
# for train [0.62508317 0.61615647 0.6207463 ]
# for entire data set [0.63102262 0.60787666 0.60858587]
print(cv_results.mean())<jupyter_output>0.6158283839346127<jupyter_text>Observations Cross validation improved AUC of Logistic Regression (L2) from  0.500668 to  0.615  Multi Label Classification using Logistic Regression and  Naive Bayes<jupyter_code># load the dataset
catdf = pd.read_json('categorized-comments.jsonl', lines = 'True')
print(catdf.head())
catdf.count()
# unique values for cat
catdf['cat'].unique()
# Checking whether categorizations are balanced or not by checking counts for each categorization
catdf.groupby('cat').count()
#Copying original data frame
catdf_orig = catdf<jupyter_output><empty_output><jupyter_text>Looks like categorizations are imbalanced, so need to add logic to balance that by adding class_weight="balanced" parameter and as it is multi-classification add multi_class = 'multinomial' for logistic regression<jupyter_code># Converting "cat" column to numerical encoding
#from sklearn.preprocessing import LabelEncoder
encoder = LabelEncoder()

cat = catdf['cat']
catdf['cat'] = encoder.fit_transform(cat)
catdf.groupby('cat').count()

catdf.head()<jupyter_output><empty_output><jupyter_text>Due to performance issues in handling 2.3M rows, using proportionate sampling to complete model fitting.<jupyter_code># Sampling huge data set and at the same time, keep the fraction of the category by using same samplng fraction across

catdf0 = catdf[catdf.cat == 0 ].sample(frac=0.05, random_state= 1)
#len(catdf0)
catdf1 = catdf[catdf.cat == 1 ].sample(frac=0.05, random_state= 1)
catdf2 = catdf[catdf.cat == 2 ].sample(frac=0.05, random_state= 1)
catdf3 = catdf[catdf.cat == 3 ].sample(frac=0.05, random_state= 1)


sampledcatdf = pd.concat([catdf0, catdf1, catdf2, catdf3])
len(sampledcatdf)

sampledcatdf.groupby('cat').count()
#Cleaning text
def textcleaning(text):
    
    text = text.lower()
    #removing \n
    text = re.sub(r'\n', '', text)
    text = re.sub(r'@\w+', '', text)
    # removing urls
    text = re.sub(r'http.?://[^\s]+[\s]?', ' ', text)
    # removing symbols and numbers
    text = re.sub('[^a-zA-Z\s]', '', text)
    # removing 3 letter words
    text = re.sub(r'(\b\w{1,3}\b)', '', text) 
    
    return text
# Applying cleaning on sampled data set
sampledcatdf['clndtxt'] = sampledcatdf['txt'].apply(textcleaning)
sampledcatdf.head()
#split data into train and test
cattext_train, cattext_test, catlabel_train, catlabel_test = model_selection.train_test_split(sampledcatdf['clndtxt']
                                                                                              ,sampledcatdf['cat'], 
                                                                                              test_size = 0.25, 
                                                                                              shuffle=True )
# Checking lenghts of test and test
print("Number of observations in train", len(cattext_train))

print("Number of observations in train", len(cattext_test))

# Creating count vectorizer objects
catcv_vect = CountVectorizer( analyzer='word', 
                                stop_words = 'english',  # removes english stop words
                                ngram_range=(2,3),       # ngrams - 2,3 
                                max_features=10000, # Had to restrict to 10000 features otherwise its running forever
                                lowercase = True,  
                                max_df = 0.5,  
                                min_df = 3)  
catcvtexttrain = catcv_vect.fit_transform(cattext_train) 
print(len(catcv_vect.get_feature_names()))
print(catcv_vect.fit_transform(cattext_train).shape) #(176061, 34779) #(176061, 41095)
# converting testing text into matric form for computation using transform
catcvtexttest = catcv_vect.transform(cattext_test)
print(catcv_vect.transform(cattext_test).shape) #(58687, 34779) # (11738, 10000)
# Confirming shapes of test and train transformations
print(catcvtexttrain.shape)
print(catcvtexttest.shape)<jupyter_output>(88030, 10000)
(29344, 10000)<jupyter_text>Logistric Refression for Categorization Data Set with Penalty L1<jupyter_code>catlogregmodel = LogisticRegression(n_jobs=-1, penalty='l1', multi_class = 'multinomial', random_state=0,
                                    class_weight="balanced", solver = 'saga')  # solver="sag" for L2

# fitting logistic regression model to trained vectors
catlogregmodel.fit(catcvtexttrain, catlabel_train)
# predict the labels on trained dataset
catlogregmodelpredtr = catlogregmodel.predict(catcvtexttrain)
# predict the labels on test dataset
catlogregmodelpredtest = catlogregmodel.predict(catcvtexttest)
# Testing total accurate predictions
print (np.sum(catlogregmodelpredtest == catlabel_test)) 
#Generate labelled performance metrics

print(metrics.classification_report(catlabel_test, catlogregmodelpredtest)) 
cataccuracy_train = metrics.accuracy_score(catlabel_train, catlogregmodelpredtr)
cataccuracy_test = metrics.accuracy_score(catlabel_test, catlogregmodelpredtest)
print(cataccuracy_train, cataccuracy_test) 
# 0.48422368009996875 0.3903561083659908
# creating a dummylist to hold model performance metrics
modelperfrows2 = []
def fn_multiclass_metrics(actual_label, predicted_label):
    """
    function that takes acutal labels and predicted labels and returns
    accuracy, auc, precision, recall and f1 scores
    average = 'weighted' for multi class classification
    """
    accuracy = accuracy_score(actual_label, predicted_label)
    precision = precision_score(actual_label, predicted_label, average = 'weighted')
    recall = recall_score(actual_label, predicted_label, average = 'weighted')
    f1 = f1_score(actual_label, predicted_label, average = 'weighted')

    return (accuracy, precision, recall, f1)
fn_multiclass_metrics(catlabel_test, catlogregmodelpredtest)

#print(fn_metrics(label_test, logregmodelpredtest))

acc, prec, recall, f1 = fn_multiclass_metrics(catlabel_test, catlogregmodelpredtest)

acc, prec, recall, f1


l1row= ['Logistic Regression (L1)', acc, prec, recall, f1]
print(l1row)

modelperfrows2.append(l1row)<jupyter_output>['Logistic Regression (L1)', 0.4155534351145038, 0.5132254291964905, 0.4155534351145038, 0.3814522093962927]<jupyter_text>Logistric Regression with L2 Pentality<jupyter_code># Creating logistic regression with penalty L2, multiclass & imbalanced classification 
catlogregmodel2 = LogisticRegression(n_jobs=-1, penalty='l2', multi_class = 'multinomial', 
                                     random_state=0, class_weight="balanced", solver = 'sag')


# fitting trained data to logistic regression with penalty L2, multiclass & imbalanced classification
catlogregmodel2.fit(catcvtexttrain, catlabel_train)
# predict the labels on trained dataset
catlogregmodelpredtr2 = catlogregmodel2.predict(catcvtexttrain)
# predict the labels on test dataset
catlogregmodelpredtest2 = catlogregmodel2.predict(catcvtexttest)
# Testing total accurate predictions
print (np.sum(catlogregmodelpredtest2 == catlabel_test)) 
#Generate labelled performance metrics

print(metrics.classification_report(catlabel_test, catlogregmodelpredtest2)) 
cataccuracy_train2 = metrics.accuracy_score(catlabel_train, catlogregmodelpredtr2)
cataccuracy_test2 = metrics.accuracy_score(catlabel_test, catlogregmodelpredtest2)
print(cataccuracy_train2, cataccuracy_test2) 
acc, prec, recall, f1 = fn_multiclass_metrics(catlabel_test, catlogregmodelpredtest2)

acc, prec, recall, f1
# creating a list of accuracy, precision, recall, f1
l2row= ['Logistic Regression (L2)', acc, prec, recall, f1]
print(l2row)

# Appending logistic regression penalty 2 model metrics
modelperfrows2.append(l2row)<jupyter_output>['Logistic Regression (L2)', 0.42097191930207195, 0.5120788817372477, 0.42097191930207195, 0.39222068844269153]<jupyter_text>Navie Bayes Algorithm for Multi Classification<jupyter_code>catNb_cv = MultinomialNB(alpha = 1) # added alpha = 1 for laplace smoothing for multi-classification

catNb_cv.fit(catcvtexttrain, catlabel_train)

# predict the labels on trained dataset
catNb_cv_predtr = catNb_cv.predict(catcvtexttrain)

# predict the labels on test dataset
catNb_cv_predtst = catNb_cv.predict(catcvtexttest)

# Calculating Accuracy scores
Acc_catNb_cv_predtr = metrics.accuracy_score(catlabel_train, catNb_cv_predtr)
Acc_catNb_cv_predtst = metrics.accuracy_score(catlabel_test, catNb_cv_predtst)

print(Acc_catNb_cv_predtr, Acc_catNb_cv_predtst) 

# Testing total accurate predictions for test
print (np.sum(catNb_cv_predtst == catlabel_test)) 

# Classification Report for test data set using Naive Bayes and CV (for vectorization) algorithms
print(metrics.classification_report(catlabel_test, catNb_cv_predtst)) 
acc, prec, recall, f1 = fn_multiclass_metrics(catlabel_test, catNb_cv_predtst)

acc, prec, recall, f1

nbrow= ['Naive Bayes', acc, prec, recall, f1]
print(nbrow)

modelperfrows2.append(nbrow)
                    
modelperfrows2<jupyter_output><empty_output><jupyter_text>Creating data Frame for reporting<jupyter_code># create data frame to hold results
multiclass_modelperf_df = pd.DataFrame()
multiclass_modelperf_df = pd.DataFrame(modelperfrows2, columns = ['Model', 'Accuracy', 'Precision', 'Recall', 'F1'])
multiclass_modelperf_df<jupyter_output><empty_output><jupyter_text>Observations: The lower accuracies for Multi-Classificaiton might have something to do     with sampling data set and restricting the maxfeatures to 10000.    Naive Bayes performed better than Logistic Regression in terms of both accuracy, recall and F1, but has lesser precision.Logistic Regression with Penalty L2 has slightly better accuracy than Penalty L1  b. Cross Validation<jupyter_code>from sklearn.model_selection import KFold, cross_val_score

# Create k-Fold cross-validation 
kf3 = KFold(n_splits=3 # 3 fold cross validation
            , shuffle=True # to shuffle observations
            ,random_state=1)


text, target = sampledcatdf['clndtxt'], sampledcatdf['cat']

# create a count vectorizer object 
count_vect = CountVectorizer(analyzer='word', 
                                stop_words = 'english',  # removes english stop words
                                ngram_range=(2,3),       # ngrams - 2,3 
                                max_features=10000, 
                                lowercase = True,  
                                max_df = 0.5,  
                                min_df = 3)

# Converting features into vectorized matrix for applying model
features = count_vect.fit_transform(text)
print(len(count_vect.get_feature_names())) 

# Conduct k-fold cross-validation
cv_results = cross_val_score(logregmodel2, # Logistic Regression with penalty L2
                            features, # Feature matrix
                            target, # Target vector
                            cv=kf3, # Cross-validation technique
                            scoring="f1_weighted", # for imbalanced multi-class returns average of all labels
                            n_jobs=-1) # Use all CPU scores
print(cv_results)
# the cross-validation score of the model’s F1 score using three-fold cross validation
print(cv_results.mean())<jupyter_output>0.39530813887574134<jupyter_text>3. Hyperparameter Selection<jupyter_code># Use scikit-learn’s GridSearchCV
from sklearn.model_selection import GridSearchCV
# Create logistic regression
logistic = LogisticRegression()

# Create range of candidate penalty hyperparameter values
penalty = ['l1', 'l2']

# Create range of candidate regularization hyperparameter values
C = np.logspace(0, 4, 10)

# Create dictionary hyperparameter candidates
hyperparameters = dict(C=C, penalty=penalty)
# Create grid search
gridsearch = GridSearchCV(logistic, hyperparameters, cv=5, n_jobs=-1, verbose=0)
text, target = df['clndtxt'], df['con']

# create a count vectorizer object 
count_vect = CountVectorizer(analyzer='word', stop_words='english', lowercase=True, min_df = 2)

# Converting features into vectorized matrix for applying model
features = count_vect.fit_transform(text)
print(len(count_vect.get_feature_names())) 
# Fit grid search
best_model = gridsearch.fit(features, target)
# View best hyperparameters
print('Best Penalty:', best_model.best_estimator_.get_params()['penalty'])
print('Best C:', best_model.best_estimator_.get_params()['C'])
# Conduct nested cross-validation and outut the average score
cross_val_score(gridsearch, features, target).mean()
# Creating data frame from cross validation of grid search with various parameters and results
results = pd.DataFrame(gridsearch.cv_results_)
results.head(10)<jupyter_output>C:\Installed\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\Installed\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\Installed\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\Installed\Anaconda3\lib\site-packages\sklearn\utils\deprec[...]
{'fraction_non_alphanumeric': 0.09238548228135579, 'fraction_numerical': 0.046878347688220426, 'mean_word_length': 4.734152334152334, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '394903', 'license': 'Unlicense', 'n_tokens_mistral': 8111, 'n_tokens_neox': 7309, 'n_words': 1852, 'path': 'Model Evaluation - Cross Validation/Model Evaluation - Cross Validation.ipynb', 'repo_name': 'rahulgupta271/rahulgupta271.github.io', 'size': '23337'}