<jupyter_start><jupyter_text>Ham or Spam?In this notebook, we use daproli to explore spam texts taken from: https://www.kaggle.com/team-ai/spam-text-message-classification<jupyter_code>import sys
sys.path.append("../..")
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import daproli as dp
df = pd.read_csv('data/spam.csv')
df.head()<jupyter_output><empty_output><jupyter_text>Let's tokenize the messages and remove all special characters and stop words.<jupyter_code>from re import sub
from nltk.corpus import stopwords

cleaner = dp.Pipeline(
    dp.Mapper(lambda word : sub('[^a-zA-Z0-9]', '', word)),
    dp.Filter(lambda word : not word.isnumeric()),
    dp.Filter(lambda word : len(word) > 1),
    dp.Filter(lambda word : word not in stopwords.words('english')),
)

tokenizer = dp.Pipeline(
    dp.Manipulator(lambda m : m.to_numpy().tolist()),
    dp.Mapper(lambda m : m.lower()),
    dp.Mapper(lambda m : m.split(' ')),
    dp.Mapper(lambda words : cleaner.transform(words), expand_args=False, verbose=1),
)

messages = tokenizer.transform(df['Message'])<jupyter_output>100%|██████████| 5572/5572 [00:10<00:00, 519.05it/s]<jupyter_text>Let's split ham and spam messages for further analysis.<jupyter_code>ham, spam = dp.Pipeline(
    dp.Combiner(lambda m, c : (m, c)),
    dp.Splitter(lambda m, c : c),
    dp.Union(
        dp.Mapper(lambda m, c : m),
        dp.Mapper(lambda m, c : m),
    ),
).transform([messages, df['Category']])<jupyter_output><empty_output><jupyter_text>Let's create the word distributions for both categories with words that come up at least 15% of the time.<jupyter_code>from itertools import chain

distribution = dp.Pipeline(
    dp.Manipulator(chain.from_iterable),
    dp.Manipulator(list),
    dp.Manipulator(np.unique, return_counts=True),
    dp.Union(
        dp.Mapper(lambda l : l, expand_args=False),
        dp.Manipulator(lambda c : (c - c.min()) / (c.max() - c.min())),
    ),
    dp.Manipulator(lambda lc : zip(*lc)),
    dp.Filter(lambda l, c : c > .15),
    dp.Manipulator(sorted, key = lambda x : x[1], reverse=True),
    dp.Expander(lambda l, c : (l,c)),
    dp.Union(dp.Manipulator(np.array), dp.Manipulator(np.array))
)
ham_bins, ham_freq = distribution.transform(ham)
spam_bins, spam_freq = distribution.transform(spam)
for cat, bins, freq in zip(['Ham', 'Spam'], [ham_bins, spam_bins], [ham_freq, spam_freq]):
    plt.figure(figsize=(20,5))
    plt.bar(np.arange(bins.shape[0]), freq)
    plt.title(f"Word distribution for {cat}")
    plt.xticks(np.arange(bins.shape[0]), bins.tolist(), rotation=90)
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15961904761904763, 'fraction_numerical': 0.015238095238095238, 'mean_word_length': 4.381147540983607, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '332844', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 993, 'n_tokens_neox': 943, 'n_words': 211, 'path': 'daproli/examples/spam.ipynb', 'repo_name': 'ermshaua/daproli', 'size': '2625'}