<jupyter_start><jupyter_text>Import necessary libraries<jupyter_code>import pandas as pd
import nltk
import re
import time
import matplotlib.pyplot as plt
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
#nltk.download('wordnet')<jupyter_output><empty_output><jupyter_text>Read data into dataframe<jupyter_code>db=pd.read_csv('data/enron_spam_ham_email_clean.csv')
db.head(10)<jupyter_output><empty_output><jupyter_text>Define processing functions<jupyter_code>lemmatizer = WordNetLemmatizer()

def regex_processing(msg):
    msg=re.sub('[^a-z\.]'," ",msg.lower()) #lowercasing and removing every character which is not an alphabet or '.'
    msg=re.sub('\s+'," ",msg)              #removing extra whitespaces
    msg=re.sub('\.+\s+\.',".",msg)         #removing double dots to avoid issues during splitting into sentences
    return msg

def sentence_dropping(msg):
    n=2                                    #attempting to drop sentences which don't have more than 2 words in it
    tmp=[sent for sent in sent_tokenize(msg) if len(sent.split())>n]
    if not tmp:
        return msg
    return ' '.join(tmp)

ref_tag={'NN':'n','NNS':'n','NNP':'n','NNPS':'n',
         'VB':'v','VBD':'v','VBG':'v','VBN':'v','VBP':'v','VBZ':'v',
         'JJ':'a','JJR':'a','JJS':'a',
         'RB':'r','RBR':'r','RBS':'r',
         'ADJ_SAT':'s'}

stop_words=set(stopwords.words('english'))

def lemma(msg,def_txt):
    tmp=[]
    for sentence in sent_tokenize(sentence_dropping(regex_processing(msg))):
        tags=nltk.pos_tag(nltk.word_tokenize(sentence))
        lemmatized=' '.join([lemmatizer.lemmatize(word) if ref_tag.get(tag,"")=="" else lemmatizer.lemmatize(word,ref_tag.get(tag)) for word,tag in tags])
        
        filtered=lemmatized
        tmp_var=' '.join([word for word in word_tokenize(lemmatized) if word not in stop_words])
        if len(tmp_var):
            filtered=tmp_var
            
        tmp.append(sentence_dropping(filtered))
    
    if not len(tmp):
        return def_txt
    
    return ' '.join(tmp)<jupyter_output><empty_output><jupyter_text>Process 'Subjects' column<jupyter_code>t=time.time()

for i in range(len(db['Subject'])):
    db.loc[i,'Subject']=lemma(db.loc[i,'Subject'],'subject')
    
print(f'Total Elapsed (Subject): {(time.time()-t)//60:.0f} m {(time.time()-t)%60:.0f} s')<jupyter_output>Total Elapsed (Subject): 1 m 38 s<jupyter_text>Process 'Messages' column<jupyter_code>t=time.time()

for i in range(len(db['Message'])):
    db.loc[i,'Message']=lemma(db.loc[i,'Message'],'message text')
    
print(f'Total Elapsed (Message): {(time.time()-t)//60:.0f} m {(time.time()-t)%60:.0f} s')

db.to_csv("data/enron_spam_ham_email_processed_v2.csv", index=False)<jupyter_output>Total Elapsed (Subject): 16 m 26 s
{'fraction_non_alphanumeric': 0.16643307638402244, 'fraction_numerical': 0.008409250175192713, 'mean_word_length': 4.346441947565543, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '383533', 'license': 'MIT', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1024, 'n_words': 193, 'path': 'Machine Learning/Spam vs Ham Emails/1_processed_enron_data.ipynb', 'repo_name': 'StandingMike/data-science', 'size': '2854'}