<jupyter_start><jupyter_text>Fetching the data from the source and looking at some higher level data<jupyter_code>import nltk
import pandas as pd
#reading the messages from the source
messages = [line.rstrip() for line in open('smsspamcollection/SMSSpamCollection')]
messages[:10]
#the messages can be red as Pandas dataframe fro easier operations
messages = pd.read_csv('smsspamcollection/SMSSpamCollection', sep='\t', names=['label', 'message'])
#messages are transformed to Pandas dataframe
messages.head()
#Some statistics related to the messages
messages.describe()
messages.groupby('label').describe()
messages['length'] = messages['message'].apply(len)
messages.head()<jupyter_output><empty_output><jupyter_text>Visualizing the data to find the most important predictor for the classification<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline
messages['length'].hist(bins=150)
messages['length'].describe()
messages.hist(column='length', by='label', bins=60, figsize=(12, 4))<jupyter_output><empty_output><jupyter_text>By looking at the above plots we can say that the text length is one of the important deciding factors  Text preprocessing before training the model<jupyter_code>import string
from nltk.corpus import stopwords
#A function that takes input as a text and removes all the punctuations and stop words and returns a clean text
def text_process(text):
    nopun = [c for c in text if c not in string.punctuation]
    nopun = ''.join(nopun)
    nopun = nopun.split()
    nostwords = [c for c in nopun if c.lower() not in stopwords.words('english')]
    return nostwords
messages['message'].apply(text_process)
messages.head()
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(analyzer=text_process)
bagofwords_messages = vectorizer.fit_transform(messages['message'])
bagofwords_messages.shape
from sklearn.feature_extraction.text import TfidfTransformer
tfidf_messages = TfidfTransformer().fit_transform(bagofwords_messages)
tfidf_messages.shape<jupyter_output><empty_output><jupyter_text>Training and Evaluating the model<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
msg_train, msg_test, label_train, label_test = train_test_split(tfidf_messages, messages['label'], test_size=0.2)
model = RandomForestClassifier()
model.fit(msg_train, label_train)
predictions = model.predict(msg_test)
from sklearn.metrics import classification_report
print(classification_report(label_test, predictions))<jupyter_output>precision    recall  f1-score   support

        ham       0.96      1.00      0.98       976
       spam       0.99      0.74      0.85       139

avg / total       0.97      0.97      0.96      1115
{'fraction_non_alphanumeric': 0.08796801163213377, 'fraction_numerical': 0.01817520901490367, 'mean_word_length': 5.761670761670762, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '667094', 'license': 'Unlicense', 'n_tokens_mistral': 861, 'n_tokens_neox': 824, 'n_words': 236, 'path': 'Spam Classifier.ipynb', 'repo_name': 'pronoy-chatterjee/spam-classifier', 'size': '2751'}