<jupyter_start><jupyter_text>Exercise Solution from Chapter 3 of Hands-On Machine Learning  Salient Points:* Uses Apache SpamAssassin's public spam-ham dataset [Link](https://homl.info/spamassassin)* Cleaned datasets in email-format* Used custom sklearn transformers for data cleaning and vectorization|   Model  | Precision | Recall ||----------|-----------|--------|| Logistic |     92%   |  96.8% || Naive Bayes|   93.9% |  97.9% |<jupyter_code>import tarfile
import os
from pathlib import Path<jupyter_output><empty_output><jupyter_text>Step 1: Downloading the Dataset -> Done  Step 2: EDA<jupyter_code>spam_data_path = Path("./data/20030228_spam.tar.bz2")
ham_data_path = Path("./data/20030228_easy_ham.tar.bz2")
# Decompression 
for path in [spam_data_path, ham_data_path]:
    tar_obj = tarfile.open(path)
    tar_obj.extractall(path="data/")
    tar_obj.close()
# Data has been decompressed
os.listdir("data/")
# Path for decompressed data
spam_path = Path("./data/spam/")
ham_path = Path("./data/easy_ham/")
# List of all spam and ham files
spam_files = [spam_path/file for file in os.listdir(spam_path)  if file != 'cmds']
ham_files = [ham_path/file for file in os.listdir(ham_path)  if file != 'cmds']
print(spam_files[:5])
print(ham_files[:5])
len(spam_files), len(ham_files)
# File format -> Email format
f = open(spam_files[0])
content = f.read()
f.close()
print(content)
# Using python's email library as it makes parsing easy
# https://docs.python.org/3/library/email.examples.html
from email import policy
from email.parser import BytesParser
def email_loader(filenames):
    parsed_email = []
    for file in filenames:
        with open(file, 'rb') as fp:
            parsed_email.append(BytesParser(policy=policy.default).parse(fp))
    return parsed_email
spam_email = email_loader(spam_files)
ham_email = email_loader(ham_files)
# Same as above, can be accessed as key value pairs
print(spam_email[0].items())
print(spam_email[0].get_content().strip())
# Checking the headers and items for the files
for key, val in zip(spam_email[23].keys(), spam_email[23].values()):
    print("{} : {}".format(key, val))<jupyter_output>Return-Path : <<EMAIL>>
Delivered-To : <EMAIL>
Received : from localhost (jalapeno [127.0.0.1])	by zzzzason.org (Postfix) with ESMTP id F3CF116F1B	for <zzzz@localhost>; Tue,  8 Oct 2002 11:02:13 +0100 (IST)
Received : from jalapeno [127.0.0.1]	by localhost with IMAP (fetchmail-5.9.0)	for zzzz@localhost (single-drop); Tue, 08 Oct 2002 11:02:13 +0100 (IST)
Received : from webnote.net (mail.webnote.net [193.120.211.219]) by    dogma.slashnull.org (8.11.6/8.11.6) with ESMTP id g989wGK10152 for    <<EMAIL>>; Tue, 8 Oct 2002 10:58:16 +0100
Received : from sanaga.camtel.cm ([195.24.194.61]) by webnote.net    (8.9.3/8.9.3) with ESMTP id KAA21887 for <<EMAIL>>;    Tue, 8 Oct 2002 10:59:00 +0100
Received : from ens.fr (host42-226.pool8173.interbusiness.it    [81.73.226.42]) by sanaga.camtel.cm with SMTP (Microsoft Exchange Internet    Mail Service Version 5.5.1960.3) id 431SJ0H6; Tue, 8 Oct 2002 06:55:54    -0000
Message-I[...]<jupyter_text>The `Content-Type` varies across the different emails and so we need to use Beautiful Soup to properly extract the content from them<jupyter_code>dict(spam_email[23].items())['Content-Type']
from collections import Counter
def email_content_type(emails):
    content_type = Counter()
    for email in emails:
        content_type[email.get_content_type()] += 1
    return content_type
print(email_content_type(ham_email))
print(email_content_type(spam_email))<jupyter_output>Counter({'text/plain': 2408, 'multipart/signed': 68, 'multipart/mixed': 10, 'multipart/alternative': 9, 'multipart/related': 3, 'multipart/report': 2})
Counter({'text/plain': 218, 'text/html': 183, 'multipart/alternative': 47, 'multipart/mixed': 43, 'multipart/related': 9})<jupyter_text>&8593; shows that spam email has a smaller proportion of data in plain text<jupyter_code># This is a good way of getting the content structures as this gives
# a more detailed output (From solutions)


# def get_email_structure(email):
#     if isinstance(email, str):
#         return email
#     payload = email.get_payload()
#     if isinstance(payload, list):
#         return "multipart({})".format(", ".join([
#             get_email_structure(sub_email)
#             for sub_email in payload
#         ]))
#     else:
#         return email.get_content_type()

# from collections import Counter

# def structures_counter(emails):
#     structures = Counter()
#     for email in emails:
#         structure = get_email_structure(email)
#         structures[structure] += 1
#     return structures
from bs4 import BeautifulSoup
from IPython.core.debugger import set_trace
# https://stackoverflow.com/questions/328356/extracting-text-from-html-file-using-python
def multi_(email):
    clean_text=""
    if len(email.get_payload()) > 100:
            clean_text = ''.join(BeautifulSoup
                (email.get_payload(), "html.parser").stripped_strings)
    else:
        for submail in email.get_payload():
            if submail.get_content_type() in ('multipart/alternative', 'multipart/related', 'multipart/mixed', 'multipart/report', 'multipart/signed'):
                clean_text += multi_(submail)
            elif submail.get_content_type() in ("application/octet-stream", "image/jpeg", "image/gif"):
                clean_text += submail.get_content_type()
            elif submail.get_content_type() in ("text/plain", "text/html"):
                clean_text += ''.join(BeautifulSoup
                    (submail.get_payload(), "html.parser").stripped_strings)
            else: continue
    return clean_text

def html2text(emails):
    data = []
    for email in emails:
        if email.get_content_type() in ("text/plain", "text/html") or len(email.get_payload()) > 100:
            clean_text = ''.join(BeautifulSoup
                (email.get_payload(), "html.parser").stripped_strings)
        elif email.get_content_type() in ('multipart/alternative', 'multipart/related', 'multipart/mixed', 'multipart/report', 'multipart/signed'):
            clean_text = multi_(email)
        else:
            clean_text = ""
            for submail in email.get_payload():
                clean_text += ''.join(BeautifulSoup
                (submail.get_payload(), "html.parser").stripped_strings)
        data.append(clean_text)
    return data
all_spam_clean = html2text(spam_email)
all_ham_clean = html2text(ham_email)
import numpy as np
from sklearn.model_selection import train_test_split

X = np.array(all_ham_clean + all_spam_clean)
y = np.array([0] * len(all_ham_clean) + [1] * len(all_spam_clean))

x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)<jupyter_output><empty_output><jupyter_text>Creating Custom Transformer<jupyter_code>from sklearn.base import TransformerMixin, BaseEstimator
from collections import Counter
import urlextract #Taken from sols as regex approach was not great
import nltk
import re

url_extractor = urlextract.URLExtract()
stemmer = nltk.PorterStemmer()

class data_prep(BaseEstimator, TransformerMixin):
    def __init__(self, lowercase=True, punctuation=True, urls=True, 
                numbers=True, stemming=True):
        self.lowercase = lowercase
        self.punctuation = punctuation
        self.urls = urls
        self.numbers = numbers
        self.stemming = True
    def fit(self, x, y=None):
        return self
    def transform(self, X, y=None):
        data = []
        for text in X:
            if self.lowercase:
                text = text.lower()
            if self.urls:
                ext_urls = list(set(url_extractor.find_urls(text)))
                ext_urls.sort(key=lambda x: len(x), reverse=True)
                for url in ext_urls:
                    text = text.replace(url, "URL")
            if self.punctuation:
                text = re.sub(r'[^\w\s]', ' ', text)
#           # From solution
            if self.numbers:
                text = re.sub(r'\d+(?:\.\d*(?:[eE]\d+))?', 'NUMBER', text)
            words = Counter(text.split())
            if self.stemming:
                word_stemmed = Counter()
                for word, count in words.items():
                    word_stemmed[stemmer.stem(word)] += count
                words = word_stemmed
            data.append(words)
        return np.array(data)
# Its working
data_prep().fit_transform(x_test[5:10])
# From solutions
from scipy.sparse import csr_matrix

class vectorizer(BaseEstimator, TransformerMixin):
    def __init__(self, vocab_size = 1000):
        self.vocab_size = vocab_size
    def fit(self, x, y=None):
        total_words = Counter()
        for text in x:
            for word, count in text.items():
                total_words[word] += count
        most_common = total_words.most_common()[:self.vocab_size]
        self.most_common = most_common
        self.vocabulary_ = {word:index+1 for index, (word, count) in enumerate(most_common)}
        return self
    def transform(self, x, y=None):
        rows = []
        cols = []
        data = []
        for index, text in enumerate(x):
            for word, count in text.items():
                rows.append(index)
                cols.append(self.vocabulary_.get(word, 0))
                data.append(count)
        return csr_matrix((data, (rows, cols)), shape=(len(x), self.vocab_size + 1))
vectorizer(vocab_size=20).fit_transform(data_prep().fit_transform(x_test[5:10]))<jupyter_output><empty_output><jupyter_text>Transforming Data for Training<jupyter_code>from sklearn.pipeline import Pipeline

data_preprocess_pipe = Pipeline([
    ("text_clean", data_prep()),
    ("text_vectorizer", vectorizer()),
])
x_train_trans = data_preprocess_pipe.fit_transform(x_train)
x_test_trans = data_preprocess_pipe.transform(x_test)
# How the transformed data in vectorized form looks
x_train_trans[0].toarray()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import precision_score, recall_score

log = LogisticRegression(random_state=42)
mnb = MultinomialNB()
def score(y_test, y_pred):
    print("Precision: {:.1f}%".format(100 * precision_score(y_test, y_pred)))
    print("Recall: {:.1f}%".format(100 * recall_score(y_test, y_pred)))
# Logistic Regression
log.fit(x_train_trans, y_train)

y_pred = log.predict(x_test_trans)
score(y_test, y_pred)
# Gaussian Naive Bayes
mnb.fit(x_train_trans, y_train)

y_pred = mnb.predict(x_test_trans)
score(y_test, y_pred)<jupyter_output>Precision: 93.9%
Recall: 97.9%
{'fraction_non_alphanumeric': 0.12502343017806936, 'fraction_numerical': 0.028209934395501405, 'mean_word_length': 3.500632644453817, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 38, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 5, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '50690', 'license': 'MIT', 'n_tokens_mistral': 3762, 'n_tokens_neox': 3481, 'n_words': 817, 'path': 'exercise/spam_classifier_chap3.ipynb', 'repo_name': 'vaibhavkumar11/hands-on-ml', 'size': '10744'}