<jupyter_start><jupyter_text>The spam fillter  1. Pre processing<jupyter_code>%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import re
import nltk.stem.porter
from scipy.io import loadmat
from sklearn import svm<jupyter_output><empty_output><jupyter_text>Output the email content<jupyter_code>!ls data
!cat data/emailSample1.txt<jupyter_output>> Anyone knows how much it costs to host a web portal ?
>
Well, it depends on how many visitors you're expecting.
This can be anywhere from less than 10 bucks a month to a couple of $100. 
You should checkout http://www.rackspace.com/ or perhaps Amazon EC2 
if youre running something big..

To unsubscribe yourself from this mailing list, send an email to:
<EMAIL><jupyter_text>strip<jupyter_code>def preProcess(email):
    email = email.lower()
    
    # Strip html tags (strings that look like <blah> where 'blah' does not
    # contain '<' or '>')... replace with a space
    email = re.sub('<[^<>]+>', ' ', email)
    
    #Any numbers get replaced with the string 'number'
    email = re.sub('[0-9]+', 'number', email)
    
    #Anything starting with http or https:// replaced with 'httpaddr'
    email = re.sub('(http|https)://[^\s]*', 'httpaddr', email)
    
    #Strings with "@" in the middle are considered emails --> 'emailaddr'
    email = re.sub('[^\s]+@[^\s]+', 'emailaddr', email)
    
    #The '$' sign gets replaced with 'dollar'
    email = re.sub('[$]+', 'dollar', email)
    
    return email<jupyter_output><empty_output><jupyter_text>2. Email to tokens list<jupyter_code>def email2TokensList(raw_email):
    stemmer = nltk.stem.porter.PorterStemmer()
    email = preProcess(raw_email)
    tokens = re.split('[ \@\$\/\#\.\-\:\&\*\+\=\[\]\?\!\(\)\{\}\,\'\"\>\_\<\;\%]', 
                      email)
    token_list = []
    for token in tokens:
        #Remove any non alphanumeric characters
        token = re.sub('[^a-zA-Z0-9]', '', token);

        #Use the Porter stemmer to stem the word
        stemmed = stemmer.stem( token )
        
        #Throw out empty tokens
        if not len(token): continue
            
        #Store a list of all unique stemmed words
        token_list.append(stemmed)
        
    return token_list<jupyter_output><empty_output><jupyter_text>3. Vocabulary list<jupyter_code>def getVocabDict(reverse=False):
    vocab_dict = {}
    with open('data/vocab.txt') as f:
        for line in f:
            val, key = line.split()
            if not reverse:
                vocab_dict[key] = int(val)
            else:
                vocab_dict[int(val)] = key
    return vocab_dict
def email2VocabIndices(raw_email, vocab_dict):
    token_list = email2TokensList(raw_email)
    indices_list = [vocab_dict[token] for token in token_list if token in vocab_dict]
    return indices_list<jupyter_output><empty_output><jupyter_text>4. Extracting features from email<jupyter_code>def email2Vector(raw_email, vocab_dict):
    n = len(vocab_dict)
    result = np.zeros((n, 1))
    vocab_indices = email2VocabIndices(raw_email, vocab_dict)
    for i in vocab_indices:
        result[i] = 1
    return result<jupyter_output><empty_output><jupyter_text>5. Process the email<jupyter_code>vocab_dict = getVocabDict()
with open('data/emailSample1.txt', 'r') as e:
    raw_email = e.read()
    vec = email2Vector(raw_email, vocab_dict)
    print(f'Length of the features vector is {len(vec)}')
    print(f'Number of non-zero entries is {np.sum(vec == 1)}')<jupyter_output>Length of the features vector is 1899
Number of non-zero entries is 45<jupyter_text>6. Training SVM for spam Classification<jupyter_code>mat = loadmat('data/spamTrain.mat')
X, y = mat['X'], mat['y']

mat = loadmat('data/spamTest.mat')
Xtest, ytest = mat['Xtest'], mat['ytest']
#m > n
X.shape, Xtest.shape 
pos = np.array([X[i] for i in range(X.shape[0]) if y[i] == 1])
neg = np.array([X[i] for i in range(X.shape[0]) if y[i] == 0])
print('Total number of training emails = ', X.shape[0])
print('Number of training spam emails = ', pos.shape[0])
print('Number of training nonspam emails = ', neg.shape[0])<jupyter_output>Total number of training emails =  4000
Number of training spam emails =  1277
Number of training nonspam emails =  2723<jupyter_text>7. Linear SVM<jupyter_code>linear_svm = svm.SVC(C=0.1, kernel='linear')

# Now we fit the SVM to our X matrix, given the labels y
linear_svm.fit( X, y.flatten() )
train_predictions = linear_svm.predict(X).reshape((y.shape[0],1))
train_acc = 100. * float(sum(train_predictions == y))/y.shape[0]
print('Training accuracy = %0.2f%%' % train_acc)

test_predictions = linear_svm.predict(Xtest).reshape((ytest.shape[0],1))
test_acc = 100. * float(sum(test_predictions == ytest))/ytest.shape[0]
print('Test set accuracy = %0.2f%%' % test_acc)<jupyter_output>Training accuracy = 99.83%
Test set accuracy = 98.90%<jupyter_text>8. Gaussion SVM<jupyter_code>gaus_svm = svm.SVC(0.3, kernel='rbf', gamma = np.power(0.3, -2))
gaus_svm.fit( X, y.flatten())
train_predictions = gaus_svm.predict(X).reshape((y.shape[0],1))
train_acc = 100. * float(sum(train_predictions == y))/y.shape[0]
print('Training accuracy = %0.2f%%' % train_acc)

test_predictions = gaus_svm.predict(Xtest).reshape((ytest.shape[0],1))
test_acc = 100. * float(sum(test_predictions == ytest))/ytest.shape[0]
print('Test set accuracy = %0.2f%%' % test_acc)<jupyter_output><empty_output><jupyter_text>9. Top predictors for spam<jupyter_code># Determine the words most likely to indicate an e-mail is a spam
# From the trained SVM we can get a list of the weight coefficients for each
# word (technically, each word index)

vocab_dict_flipped = getVocabDict(reverse=True)

#Sort indicies from most important to least-important (high to low weight)
sorted_indices = np.argsort( linear_svm.coef_, axis=None )[::-1]
print("The 15 most important words to classify a spam e-mail are:")
print([ vocab_dict_flipped[x] for x in sorted_indices[:15] ])
print("")
print("The 15 least important words to classify a spam e-mail are:")
print([ vocab_dict_flipped[x] for x in sorted_indices[-15:] ])
print("")

# Most common word (mostly to debug):
most_common_word = vocab_dict_flipped[sorted_indices[0]]
print('# of spam containing \"%s\" = %d/%d = %0.2f%%'% \
    (most_common_word, sum(pos[:,1190]),pos.shape[0],  \
     100.*float(sum(pos[:,1190]))/pos.shape[0]))
print('# of NON spam containing \"%s\" = %d/%d = %0.2f%%'% \
    (most_common_word, sum(neg[:,1190]),neg.shape[0],      \
     100.*float(sum(neg[:,1190]))/neg.shape[0]))<jupyter_output>The 15 most important words to classify a spam e-mail are:
['otherwis', 'clearli', 'remot', 'gt', 'visa', 'base', 'doesn', 'wife', 'previous', 'player', 'mortgag', 'natur', 'll', 'futur', 'hot']

The 15 least important words to classify a spam e-mail are:
['http', 'toll', 'xp', 'ratio', 'august', 'unsubscrib', 'useless', 'numberth', 'round', 'linux', 'datapow', 'wrong', 'urgent', 'that', 'spam']

# of spam containing "otherwis" = 804/1277 = 62.96%
# of NON spam containing "otherwis" = 301/2723 = 11.05%<jupyter_text>10. What about linear logisitic regression<jupyter_code>from sklearn import metrics
from sklearn.linear_model import LogisticRegression
print(metrics.classification_report(ytest, test_predictions))
logit = LogisticRegression()
logit.fit(X, y)
pred = logit.predict(Xtest)
print(metrics.classification_report(ytest, pred))<jupyter_output>precision    recall  f1-score   support

           0       1.00      0.99      1.00       692
           1       0.99      0.99      0.99       308

    accuracy                           0.99      1000
   macro avg       0.99      0.99      0.99      1000
weighted avg       0.99      0.99      0.99      1000
{'fraction_non_alphanumeric': 0.1430241621200312, 'fraction_numerical': 0.029877890361132763, 'mean_word_length': 3.839094908862351, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 53, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '281027', 'license': 'Apache-2.0', 'n_tokens_mistral': 2867, 'n_tokens_neox': 2705, 'n_words': 711, 'path': 'StudyNotesOfML/6. SVM/2 Spam.ipynb', 'repo_name': 'yang-233/ML', 'size': '7724'}