<jupyter_start><jupyter_text>Import Required Libraries<jupyter_code>import pandas as pd
import numpy as np
from textblob import TextBlob
from sklearn.feature_extraction.text import CountVectorizer,TfidfTransformer
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import warnings
warnings.filterwarnings('ignore')
import dill<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>data = pd.read_csv('data.csv',sep='\t',header=None)
messages = pd.DataFrame(list(zip(data[0].values,data[1].values)),columns=['Class','Message'])
messages.head()
print(messages.groupby('Class').count())
print("\nTotal Message",len(messages))<jupyter_output>Message
Class         
ham       4825
spam       747

Total Message 5572<jupyter_text>Data Preprocessing<jupyter_code># Split Mesaage to words
SplitIntoWords = lambda message: TextBlob(message.lower()).words
print(messages.Message.head().apply(SplitIntoWords))
# Convert words to base form
WordsIntoBaseForm = lambda message: [word.lemma for word in SplitIntoWords(message)]
# Convert each messafe into a vector
trainingVector = CountVectorizer(analyzer=WordsIntoBaseForm,stop_words = 'english').fit(messages['Message'])
message10 = trainingVector.transform([messages['Message'][9]])
print(message10)
# Print message #10 for comparison
print(messages['Message'][9])
# Identify repeated words
print ('First word that appears twice:',trainingVector.get_feature_names()[3433])
print ('Word that appears three times:',trainingVector.get_feature_names()[5182])
# Bag of words for whole training data
messagesBagOfWords = trainingVector.transform(messages["Message"])
# Weight of words int the entire training data
TfidfTransformer = TfidfTransformer()
messagesTfidf = TfidfTransformer.fit_transform(messagesBagOfWords)
example = ["england"]
trainingVector.transform(example)<jupyter_output><empty_output><jupyter_text>Split Data into train and test<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(messagesTfidf,messages['Class'].values)
X_train.shape<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code>classifiers=[]
# Using GaussianNB
from sklearn.naive_bayes import GaussianNB

spamDetector = GaussianNB().fit(X_train.toarray(),y_train)
y_pred = spamDetector.predict(X_test.toarray())
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
classifiers.append(['GaussianNB',accuracy])
# Using MultinomialNB
from sklearn.naive_bayes import MultinomialNB

spamDetector = MultinomialNB().fit(X_train,y_train)
y_pred = spamDetector.predict(X_test)
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
classifiers.append(['MultinomialNB',accuracy])
# Using BernoulliNB
from sklearn.naive_bayes import BernoulliNB

spamDetector = BernoulliNB().fit(X_train,y_train)
y_pred = spamDetector.predict(X_test)
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
classifiers.append(['BernoulliNB',accuracy])
# Using Random Forest
from sklearn.ensemble import RandomForestClassifier

spamDetector = RandomForestClassifier().fit(X_train,y_train)
y_pred = spamDetector.predict(X_test)
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
classifiers.append(['RandomForestClassifier',accuracy])
# Using SVC
from sklearn.svm import SVC

spamDetector = SVC().fit(X_train,y_train)
y_pred = spamDetector.predict(X_test)
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
classifiers.append(['SVC',accuracy])
# Using LinearSVC
from sklearn.svm import LinearSVC

spamDetector = LinearSVC().fit(X_train,y_train)
y_pred = spamDetector.predict(X_test)
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
classifiers.append(['LinearSVC',accuracy])
# Using SGDClassifier
from sklearn.linear_model import SGDClassifier

spamDetector = SGDClassifier().fit(X_train,y_train)
y_pred = spamDetector.predict(X_test)
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
classifiers.append(['SGDClassifier',accuracy])
classifiers
import matplotlib.pyplot as plt

objects = [i[0] for i in classifiers]
y_pos = np.arange(len(objects))
performance = [(round(i[1],4)*100) for i in classifiers]

plt.barh(y_pos, performance, align='center', alpha=0.8)
for i, v in enumerate(performance):
    plt.text(v-15 , i-0.1 , str(v)+"%", color='white',fontweight='bold')
plt.yticks(y_pos, objects)
plt.xlabel('Accuaracy')
plt.title('Performance')
plt.show()<jupyter_output><empty_output><jupyter_text>Save the best model<jupyter_code>import pickle

# Dump the model
spamDetector = SGDClassifier().fit(X_train,y_train)
y_pred = spamDetector.predict(X_test)
accuracy = accuracy_score(y_pred,y_test)
print("Accuracy is ",accuracy)
pickle.dump(spamDetector,open('model.sav','wb'))

import dill
# Dump the trainingVector
with open("trainingVector", "wb") as dill_file:
    dill.dump(trainingVector, dill_file)
#pickle.dump(trainingVector,open('trainingVector.sav','wb'))
example = ['England v Macedonia - dont miss the goals/team news. Txt ENGLAND to 99999']
# Result
checkResult = spamDetector.predict(trainingVector.transform(example))
print ('The message [',example[0],'] has been classified as', checkResult)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11966463414634146, 'fraction_numerical': 0.008955792682926829, 'mean_word_length': 8.92249527410208, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '454908', 'license': 'MIT', 'n_tokens_mistral': 1831, 'n_tokens_neox': 1744, 'n_words': 344, 'path': 'Spam Detection/Model Selection.ipynb', 'repo_name': 'ratansingh98/Email-Filtering', 'size': '5248'}