<jupyter_start><jupyter_text>Preprocessing Notebook  Installs<jupyter_code>!pip install -U spacy
!python -m spacy download en_core_web_sm<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>from google.colab import drive
import pandas as pd
import spacy
import regex as re
import nltk
from nltk.corpus import stopwords
from tqdm.notebook import tqdm
nltk.download('stopwords')<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.<jupyter_text>Preprocessing of email text<jupyter_code>drive.mount("/content/drive")
file = open('/content/drive/MyDrive/Mails.csv',mode='r', encoding='utf-8')
lines = file.read().split('!#ENDOFLINE#!')
file.close()

# Split the data
for i in range(len(lines)):
    lines[i] = lines[i].split('!#!')
    for j in range(len(lines[i])):
        lines[i][j] = lines[i][j].strip()
        
# Load it to a data frame
df = pd.DataFrame(lines)
df.columns = df.iloc[0]
df = df[1:]

# Small bugfix (work around)
df = df.rename(columns={"﻿Subject" : "Subject"})
nlp = spacy.load('en_core_web_sm')
COLUMN = ['Body']
#symbols = re.compile('[/(){}\[\]\|@,;:]')
#symbols = ['<', '>', '|', ':', '/', '(', ')', '[', ']', ',', '.', '\\', ' ', '_', '-', '\'', '\"', '!', '?', ';', '&']
STOPWORDS = set(stopwords.words('english'))

df["Body_processed"] = ""


def prepare(data_frame):
    for index, row in tqdm(data_frame.iterrows(), total=data_frame.shape[0]):
        for x in COLUMN:
            if not pd.isnull(data_frame[x][index]):
                # Add title to body
                text = row['Subject'] + " " + row["SenderName"] + " " + row["CCAddresses"] + " " + row['Body']
                # Lemmatize
                doc = nlp(text)
                result = []
                for token in doc:
                    str_token = str(token)
                    if not (str_token.startswith("http://") or str_token.startswith("https://") or len(str_token.strip()) <= 1 or '\\n' in str_token or '\n' in str_token):
                        lemma = token.lemma_.lower()
                        if not lemma in STOPWORDS:
                            result.append(lemma)
                
                result.append(row["SenderAddress"])
                data_frame['Body_processed'][index] = result
    return data_frame
   
df = prepare(df)
df.to_csv('/content/drive/MyDrive/Mails_prepared.csv')
df["Body_processed"][10]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17774154097024053, 'fraction_numerical': 0.0028536485935589076, 'mean_word_length': 2.945337620578778, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '268031', 'license': 'MIT', 'n_tokens_mistral': 871, 'n_tokens_neox': 842, 'n_words': 163, 'path': 'Prototyping/Preprocessing.ipynb', 'repo_name': 'MerlinKoehler/MailClassification', 'size': '2453'}