<jupyter_start><jupyter_text>Load data<jupyter_code>PATH = 'data/'
fname = f"{PATH}raw_emails.csv"
df = pd.read_csv(fname)<jupyter_output><empty_output><jupyter_text>Clean base data<jupyter_code>df.head()
results_dict = {}
for row in tqdm_notebook(df.iterrows()):
    message_id, contents = clean_entry(*row)
    results_dict[message_id] = contents
%%time
# Takes ~1:30
results = pd.DataFrame(results_dict).T
# Re-organize the resulting dataframe
new_cols = ['date', 'from_email', 'message_id', 'subject', 'to_email',
            'from_name', 'to_name', 'bcc_name', 'cc_name', 'fname_prefix',
            'orig_fname', 'orig_i', 'email_content']
results.columns = new_cols
results.index.name = 'message_id'

results.sort_values('orig_i', inplace=True)
results.drop(['message_id', 'orig_i'], axis=1, inplace=True)

results = results[['date', 'from_email', 'to_email', 'subject',
                   'email_content', 'from_name', 'to_name', 'cc_name',
                   'bcc_name', 'orig_fname', 'fname_prefix']]
out_path = f"{PATH}cleaned_emails.csv"
results.to_csv(out_path)
results.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16264521894548703, 'fraction_numerical': 0.0035746201966041107, 'mean_word_length': 5.549707602339181, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '5965', 'license': 'MIT', 'n_tokens_mistral': 423, 'n_tokens_neox': 418, 'n_words': 68, 'path': 'preprocessing.ipynb', 'repo_name': 'alexcoda/independent_study', 'size': '1119'}