<jupyter_start><jupyter_text>**1.Defining the Question**  **a) Specifying the Data Analytic Question** Implementing Naive Bayes Classiffier to predict whether an email is a spam or not.  **b) Defining the Metric for Success** To successfully implement Naive Bayes classier to make effective predictions with a high accuracy score.  **c.)Understanding the Context**  **d.)Recording the Experimental Design** 1.Defining the Question2.Reading the Data.3.Checking the Data.4.Data Cleaning5.Performing EDA6.Building Naive Bayes Theorem7.Evaluation of the solution8.Challenging the solution9.Conclusion  **e) Data Relevance** This will be discussed after the analysis.  **2.Reading the Data** **Importing the Libraries**<jupyter_code># Let us first import all the libraries we will need for our analysis
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib
import re
from matplotlib import pyplot as plt
%matplotlib inline
from pandas_profiling import ProfileReport
from scipy import stats
from scipy.stats import norm
import math
import statsmodels.api as sm
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, KFold, cross_val_score
from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, f1_score<jupyter_output>/usr/local/lib/python3.7/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
  import pandas.util.testing as tm<jupyter_text>**Loading the Data**<jupyter_code># Loading the Data from the source i.e. csv

spam_columns=spam_columns = ['word_freq_make', 'word_freq_address', 'word_freq_all', 'word_freq_3d', 'word_freq_our',
'word_freq_over', 'word_freq_remove', 'word_freq_internet', 'word_freq_order', 'word_freq_mail',
'word_freq_receive','word_freq_will','word_freq_people','word_freq_report','word_freq_addresses',
'word_freq_free','word_freq_business','word_freq_email','word_freq_you','word_freq_credit','word_freq_your',
'word_freq_font','word_freq_000','word_freq_money','word_freq_hp','word_freq_hpl','word_freq_george',
'word_freq_650','word_freq_lab','word_freq_labs','word_freq_telnet','word_freq_857','word_freq_data',
'word_freq_415','word_freq_85','word_freq_technology','word_freq_1999','word_freq_parts','word_freq_pm',
'word_freq_direct','word_freq_cs','word_freq_meeting','word_freq_original','word_freq_project',
'word_freq_re','word_freq_edu','word_freq_table','word_freq_conference','char_freq_;','char_freq_(',
'char_freq_[','char_freq_!','char_freq_$','char_freq_#','capital_run_length_average',
'capital_run_length_longest','capital_run_length_total','spam']



spam = pd.read_csv('/content/spambase (1).data', names=spam_columns)
spam.head()<jupyter_output><empty_output><jupyter_text>3. Checking the Data<jupyter_code># Determining the no. of records in our dataset
#
spam.shape
# Previewing the top of our dataset
#
spam.head()
# Previewing the bottom of our dataset
#
spam.tail() 
# Checking whether each column has an appropriate datatype
#
spam.dtypes
spam.info()
spam.columns
#checking the summary statistics
spam.describe()<jupyter_output><empty_output><jupyter_text>4. External Data Source Validation  a.Validation  5. Tidying the Dataset<jupyter_code># Checking for duplicates

spam.duplicated().sum()

#we have some duplicates in our dataframe but we will keep them
spam.isnull().sum()<jupyter_output><empty_output><jupyter_text>6. Exploratory Analysis<jupyter_code># Ploting the univariate summaries and recording our observations
def uni_stats(df):
  out_df = pd.DataFrame(columns=['Count', 'Nulls', 'Unique', 'Mode', 'Mean', 'Min','25%', 'Median', '75%', 'Max', 'Std_dev'])


  for col in df:
    if pd.api.types.is_numeric_dtype(df[col]):
      out_df.loc[col] = [df[col].count(), df[col].isnull().sum(), df[col].nunique(), df[col].mode().values[0], df[col].mean(), df[col].min(), 
                         df[col].quantile(0.25), df[col].median(), df[col].quantile(0.75), df[col].max(), df[col].std()]

    else:
     out_df.loc[col] = [df[col].count(), df[col].isnull().sum(), df[col].nunique(),df[col].mode().values[0],'-', '-', '-', '-', 
                        '-', '-', '-']

  return out_df

uni_stats(spam)
#Checking for correlation among variables
spam.corr()

#count plot for Spam
sns.countplot(x='spam',data=spam, hue='spam')
plt.title('Proportion of Ham/Spam emails')
plt.legend(['Ham', 'Spam'])
plt.show()
#plotting histograms to visualize patterns in the data
spam.hist(figsize = (30,20))<jupyter_output><empty_output><jupyter_text>**7.Implementing the Solution**  **Naive Bayes Classiffication** **Multinomial NB Classiffier** **80-20 Split**<jupyter_code># Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.20, random_state=0)
# 
# Training our model
from sklearn.naive_bayes import MultinomialNB  
model = MultinomialNB().fit(X_train, y_train)
# 
# Predicting our test predictors
from sklearn.metrics import mean_squared_error

y_pred = model.predict(X_test)
# 
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8121606948968513
[[454  84]
 [ 89 294]]
              precision    recall  f1-score   support

           0       0.84      0.84      0.84       538
           1       0.78      0.77      0.77       383

    accuracy                           0.81       921
   macro avg       0.81      0.81      0.81       921
weighted avg       0.81      0.81      0.81       921<jupyter_text>**70-30 split**<jupyter_code># Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.30, random_state=0)
# 
# Training our model
from sklearn.naive_bayes import MultinomialNB  
model = MultinomialNB().fit(X_train, y_train)
# 
# Predicting our test predictors
from sklearn.metrics import mean_squared_error

y_pred = model.predict(X_test)
# 
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8095582910934106
[[699 123]
 [140 419]]
              precision    recall  f1-score   support

           0       0.83      0.85      0.84       822
           1       0.77      0.75      0.76       559

    accuracy                           0.81      1381
   macro avg       0.80      0.80      0.80      1381
weighted avg       0.81      0.81      0.81      1381<jupyter_text>**60-40 split**<jupyter_code># Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.40, random_state=0)
# 
# Training our model
from sklearn.naive_bayes import MultinomialNB  
model = MultinomialNB().fit(X_train, y_train)
# 
# Predicting our test predictors
from sklearn.metrics import mean_squared_error

y_pred = model.predict(X_test)
# 
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8098859315589354
[[952 145]
 [205 539]]
              precision    recall  f1-score   support

           0       0.82      0.87      0.84      1097
           1       0.79      0.72      0.75       744

    accuracy                           0.81      1841
   macro avg       0.81      0.80      0.80      1841
weighted avg       0.81      0.81      0.81      1841<jupyter_text>Multinomial classiffier performs equally on all the splits with an accuracy score of 81% **Gaussian Classifier** **80-20 split**<jupyter_code># Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.20, random_state=0)
# 
# Training our model
from sklearn.naive_bayes import GaussianNB 
clf = GaussianNB()
model = clf.fit(X_train, y_train) 

# Predicting our test predictors
y_pred = model.predict(X_test)

# evaluating the model
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8067318132464713
[[386 152]
 [ 26 357]]
              precision    recall  f1-score   support

           0       0.94      0.72      0.81       538
           1       0.70      0.93      0.80       383

    accuracy                           0.81       921
   macro avg       0.82      0.82      0.81       921
weighted avg       0.84      0.81      0.81       921<jupyter_text>**60-40 split**<jupyter_code># Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.40, random_state=0)
# 
# Training our model
from sklearn.naive_bayes import GaussianNB 
clf = GaussianNB()
model = clf.fit(X_train, y_train) 

# Predicting our test predictors
y_pred = model.predict(X_test)

# evaluating the model
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8124547429398986
[[597 225]
 [ 34 525]]
              precision    recall  f1-score   support

           0       0.95      0.73      0.82       822
           1       0.70      0.94      0.80       559

    accuracy                           0.81      1381
   macro avg       0.82      0.83      0.81      1381
weighted avg       0.85      0.81      0.81      1381<jupyter_text>**70-30 split**<jupyter_code># Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.30, random_state=0)
# 
# Training our model
from sklearn.naive_bayes import GaussianNB 
clf = GaussianNB()
model = clf.fit(X_train, y_train) 

# Predicting our test predictors
y_pred = model.predict(X_test)

# evaluating the model
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8124547429398986
[[597 225]
 [ 34 525]]
              precision    recall  f1-score   support

           0       0.95      0.73      0.82       822
           1       0.70      0.94      0.80       559

    accuracy                           0.81      1381
   macro avg       0.82      0.83      0.81      1381
weighted avg       0.85      0.81      0.81      1381<jupyter_text>So far the Gaussian clasifier since it gave us an accuracy score of 81%across all the splits.We will therefore perform standardization and hyperparameter tuning on it to see if the accuracy improves.<jupyter_code># Getting the independent and dependent variables

X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.40, random_state=0)

# Feature Scaling

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)

X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)

# Training our model
from sklearn.naive_bayes import GaussianNB 
clf = GaussianNB()
model = clf.fit(X_train, y_train) 

# Predicting our test predictors
y_pred = model.predict(X_test)

# evaluating the model
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8147745790331342
[[805 292]
 [ 49 695]]
              precision    recall  f1-score   support

           0       0.94      0.73      0.83      1097
           1       0.70      0.93      0.80       744

    accuracy                           0.81      1841
   macro avg       0.82      0.83      0.81      1841
weighted avg       0.85      0.81      0.82      1841<jupyter_text>Performing standardization on the model did not have any effect .Lets tune the parameters and see if the accuracy improves **Hyperparameter Tuning**<jupyter_code># output the parameters to be hypertuned
GaussianNB()

#The var_smoothing parameter's default value is  10−9 . 
#We will conduct the grid search in the "logspace", that is, 
#we will search over the powers of 10. We will start with  100  and end with  10−9  
#and we will try 100 different values. For this search, we will use the logspace function in the numpy module.
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import PowerTransformer
from sklearn.model_selection import RepeatedStratifiedKFold

np.random.seed(999)

nb_classifier = GaussianNB()

params_NB = {'var_smoothing': np.logspace(0,-9, num=100)}

cv_method = RepeatedStratifiedKFold(n_splits=15, 
                                    n_repeats=3, 
                                    random_state=0)

gs_NB = GridSearchCV(estimator=nb_classifier, 
                     param_grid=params_NB, 
                     cv=cv_method,
                     verbose=1, 
                     scoring='accuracy')

gs_NB.fit(X_train, y_train)

gs_NB.best_params_
print(gs_NB.best_params_)
print(gs_NB.best_score_)
print(gs_NB.best_estimator_)
# applying best params
# Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

# Splitting the data to train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.40, random_state=0)
# 
# Training our model
from sklearn.naive_bayes import GaussianNB 
clf = GaussianNB(priors=None, var_smoothing= 1.232846739442066e-06)
model = clf.fit(X_train, y_train) 

# Predicting our test predictors
y_pred = model.predict(X_test)

# evaluating the model
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>0.8609451385116784
[[926 171]
 [ 85 659]]
              precision    recall  f1-score   support

           0       0.92      0.84      0.88      1097
           1       0.79      0.89      0.84       744

    accuracy                           0.86      1841
   macro avg       0.85      0.86      0.86      1841
weighted avg       0.87      0.86      0.86      1841<jupyter_text>We can observe that Tuning the parameters on the Gaussian Naive Bayes Classifier,te accuracy score on the 60-40 split increased from 81% to 86%. **Improving Model Prerformance LDA**<jupyter_code># Getting the independent and dependent variables
# 
X = spam.drop(columns = ['spam'], axis=1)
y = spam.spam

#Train Test Split (60-40)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4, random_state=0)

# Feature Scaling

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)

X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)

# Peforming LDA
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
lda = LDA(n_components=40)
X_train = lda.fit_transform(X_train, y_train)
X_test = lda.transform(X_test)

# Training our model
from sklearn.naive_bayes import GaussianNB 
clf = GaussianNB()
model = clf.fit(X_train, y_train) 

# Predicting our test predictors
y_pred_lda = model.predict(X_test)

# evaluating the model
from sklearn.metrics import accuracy_score
import scipy.stats as stats
print(accuracy_score(y_test,y_pred_lda))
print(confusion_matrix(y_test, y_pred_lda))
print(classification_report(y_test, y_pred_lda))<jupyter_output>0.8940793047256925
[[1040   57]
 [ 138  606]]
              precision    recall  f1-score   support

           0       0.88      0.95      0.91      1097
           1       0.91      0.81      0.86       744

    accuracy                           0.89      1841
   macro avg       0.90      0.88      0.89      1841
weighted avg       0.90      0.89      0.89      1841
{'fraction_non_alphanumeric': 0.09606085623304635, 'fraction_numerical': 0.05773086448873688, 'mean_word_length': 3.4015053205294574, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '457939', 'license': 'MIT', 'n_tokens_mistral': 6349, 'n_tokens_neox': 5690, 'n_words': 1457, 'path': 'NB_DSC2_WK4_IP_Iyline_Sigey_Moringa_School.ipynb', 'repo_name': 'iyline-sigey/KNN-and-Naive-Bayes-Classification-Models', 'size': '16958'}