<jupyter_start><jupyter_text>Model<jupyter_code>import pandas as pd

import numpy as np
import sklearn.utils

import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.corpus import words

from sklearn.feature_extraction.text import TfidfVectorizer
import joblib

import matplotlib.pyplot as plt

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline    
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score
from sklearn.metrics import precision_score, recall_score, plot_confusion_matrix, classification_report, accuracy_score, f1_score
from sklearn import metrics
# import dataframe
df = joblib.load("exports/dataframe.sav")<jupyter_output><empty_output><jupyter_text>Build vocabularyBuild the vocabulary over all words of our dataset, because the vocabulary can not be changed after the model is trained.<jupyter_code>collected_words = set()
for text in df["clean_text"]:
    for word in text:
        collected_words.add(word)

wordlist_nltk = words.words()

wordlist_github = [] # https://github.com/dwyl/english-words
with open("resources/words-dwyl-github.txt", "r") as f:
    for line in f.readlines():
        wordlist_github.append(line[:-1])

wordlist_github_nltk_combined = wordlist_github + wordlist_nltk

wordlist_github_nltk_combined_lower = [x.lower() for x in wordlist_github_nltk_combined]
intersection_github_nltk_combined_lower = collected_words.intersection(wordlist_github_nltk_combined_lower)
vocabulary = intersection_github_nltk_combined_lower
len(vocabulary)

joblib.dump(vocabulary, "exports/vocab.sav")<jupyter_output><empty_output><jupyter_text>Prepare dataset parts<jupyter_code>len(df)
df_spam = df[df["spam"] == 1]
df_ham = df[df["spam"] == 0]
len(df_spam)
len(df_ham)
# use 25k for balanced dataset
amount = 25000
# reshuffle dataframes (currently spam and ham are strictly separated)
df_spam = sklearn.utils.shuffle(df_spam)
df_ham = sklearn.utils.shuffle(df_ham)

# fix indices
df_spam = df_spam.reset_index(drop=True)
df_ham = df_ham.reset_index(drop=True)
# create balanced df (25k spam and 25k ham mails) for training the model
df_model_train = df_spam[:amount].append(df_ham[:amount])
df_model_train = sklearn.utils.shuffle(df_model_train)
df_model_train = df_model_train.reset_index(drop=True)
# use the rest of df_spam and df_ham but keep the new df (df_model_test) balanced
df_model_test = df_spam[amount:28410].append(df_ham[amount:28410])
df_model_test = sklearn.utils.shuffle(df_model_test)
df_model_test = df_model_test.reset_index(drop=True)
# used for cross-validation, contains train and test dataset, balanced
df_balanced = df_model_train.append(df_model_test)
df_balanced = sklearn.utils.shuffle(df_balanced)
df_balanced = df_balanced.reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Estimator test<jupyter_code>corpus = []
for text in df_model_train["clean_text"]:
    msg = ' '.join([row for row in text])
    corpus.append(msg)
tfidf = TfidfVectorizer(vocabulary=vocabulary)
x_train = tfidf.fit_transform(corpus).toarray()
y_train = df_model_train["spam"]
corpus = []
for text in df_model_test["clean_text"]:
    msg = ' '.join([row for row in text])
    corpus.append(msg)
x_test = tfidf.fit_transform(corpus).toarray()
y_test = df_model_test["spam"]
import time
# try different estimators for the model 
classifiers = [MultinomialNB(), 
               RandomForestClassifier(),
               KNeighborsClassifier(),
               #SVC()
              ]

for cls in classifiers:
    time_start = time.time()
    cls.fit(x_train, y_train)
    time_end = time.time()
    print("Estimator:", cls)
    print("Train-Duration:", time_end - time_start, "s")
    score = cls.score(x_test, y_test)
    print("Accuracy:", score, "\n")
# SVC() will no longer be considered due to extremly long run time
# after 8 hours it still hasn't finished ...
#classifiers = classifiers[:-1] # remove SVC() from list (if not uncomment in classifiers)
# RandomForest worked the best (highest accuracy) and was the most efficient one for our tests<jupyter_output><empty_output><jupyter_text>Cross-validation<jupyter_code>corpus = []
for text in df_balanced["clean_text"]:
    msg = ' '.join([row for row in text])
    corpus.append(msg)
tfidf = TfidfVectorizer(vocabulary=vocabulary)
x_all = tfidf.fit_transform(corpus).toarray()
y_all = df_balanced["spam"]
x_all
import time
# cross-validation using only clean_text with TFIDF as features
for cls in classifiers:
    time_start = time.time()
    cv_score = cross_val_score(cls, x_all,y_all,scoring="accuracy", cv=10)
    time_end = time.time()
    print("Estimator:", cls)
    print("Crossval-Duration:", time_end - time_start, "s")
    mean_score = cv_score.mean()
    print("Mean accuracy:", mean_score, "\n")
# so looks like RandomForest performs best
# lets try to add more features and do the cross-validation again to see if this improves the accuracy
x_all = np.append(x_all, df_balanced[["tokens", "chars"]].to_numpy(), axis=1)
x_all
# cross-validation using clean_text with TFIDF AND number of tokens and number of chars as features
for cls in classifiers:
    time_start = time.time()
    cv_score = cross_val_score(cls, x_all,y_all,scoring="accuracy", cv=10)
    time_end = time.time()
    print("Estimator:", cls)
    print("Crossval-Duration:", time_end - time_start, "s")
    mean_score = cv_score.mean()
    print("Mean accuracy:", mean_score, "\n")
# RandomForest is still the best
# by using more features (#tokens, #chars) we could improve the accuracy
# RandomForest only clean_text: 0.9771383315733896
# RandomForest clean_text, #chars, #tokens: 0.9793206617388244<jupyter_output><empty_output><jupyter_text>Export final model<jupyter_code># ours experiments showed that the best estimator is RandomForest with TFIDF AND #tokens, #chars
corpus = []
for text in df_model_train["clean_text"]:
    msg = ' '.join([row for row in text])
    corpus.append(msg)
tfidf = TfidfVectorizer(vocabulary=vocabulary)
x_train = tfidf.fit_transform(corpus).toarray()
x_train = np.append(x_train, df_model_train[["tokens", "chars"]].to_numpy(), axis=1)
y_train = df_model_train["spam"]
corpus = []
for text in df_model_test["clean_text"]:
    msg = ' '.join([row for row in text])
    corpus.append(msg)
x_test = tfidf.fit_transform(corpus).toarray()
x_test = np.append(x_test, df_model_test[["tokens", "chars"]].to_numpy(), axis=1)
y_test = df_model_test["spam"]
import time
classifiers = [RandomForestClassifier()]

for cls in classifiers:
    time_start = time.time()
    cls.fit(x_train, y_train)
    time_end = time.time()
    print("Estimator:", cls)
    print("Train-Duration:", time_end - time_start, "s")
    score = cls.score(x_test, y_test)
    print("Accuracy:", score, "\n")
# export model
joblib.dump(classifiers[0], "exports/model.sav")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10844394238915561, 'fraction_numerical': 0.009743010449025698, 'mean_word_length': 5.61344537815126, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '658313', 'license': 'MIT', 'n_tokens_mistral': 2505, 'n_tokens_neox': 2422, 'n_words': 584, 'path': 'model.ipynb', 'repo_name': 'konstantingoretzki/spamdetection', 'size': '7082'}