<jupyter_start><jupyter_text>Support Vector MachinesUse support vector machines(SVMs) to build a spam classifer.<jupyter_code>import numpy as np
from scipy.io import loadmat
import matplotlib.pyplot as plt
from sklearn import svm
%matplotlib inline<jupyter_output><empty_output><jupyter_text>1 SVMs with linear kernelVisualizing the dataset.<jupyter_code>data = loadmat('data/ex6data1.mat')
X_raw = data['X']
y_raw = data['y'].ravel()

X_positive = X_raw[y_raw == 1]
X_negative = X_raw[y_raw == 0]
print(X_positive.shape)
print(X_negative.shape)
def plot_raw(positive, negative):
    fig, ax = plt.subplots(figsize=(12, 8))
    ax.scatter(positive[:, 0], positive[:, 1], s=50, marker='x', label='Positive')
    ax.scatter(negative[:, 0], negative[:, 1], s=50, marker='o', label='Negative')
    ax.legend()
plot_raw(X_positive, X_negative)
def plot_decision_boundary(w, b, X_raw, C):
    '''
    w1x1 + w2x2 + b = 0 -> x2 = -w1x1/w2 - b/w2
    '''
    X = np.linspace(X_raw[:, 0].min(), X_raw[:, 0].max(), 100)
    y = -w[0] / w[1] * X - b / w[1]
    
    fig, ax = plt.subplots(figsize=(12, 8))
    ax.plot(X, y, 'r', label='Decision boundary')
    ax.scatter(X_positive[:, 0], X_positive[:, 1], s=50, marker='x', label='Positive')
    ax.scatter(X_negative[:, 0], X_negative[:, 1], s=50, marker='o', label='Negative')
    ax.set_title('SVMs(C={}) Decision boundary'.format(C))
    ax.legend()
def plot_confidence(clf, X_raw):
    confidence = clf.decision_function(X_raw)

    fig, ax = plt.subplots(figsize=(12, 8))
    ax.scatter(X_raw[:, 0], X_raw[:, 1], s=50, c=confidence, cmap='seismic')
    ax.set_title('Decision confidence')<jupyter_output><empty_output><jupyter_text>Try $C=1$<jupyter_code>clf = svm.LinearSVC(C=1, loss='hinge', max_iter=3000)
clf.fit(X_raw, y_raw)
w = clf.coef_.ravel()
b = clf.intercept_
clf.score(X_raw, y_raw)
plot_decision_boundary(w, b, X_raw, 1)
plot_confidence(clf, X_raw)<jupyter_output><empty_output><jupyter_text>Try $C=1000$<jupyter_code>clf = svm.LinearSVC(C=1000, loss='hinge', max_iter=200000)
clf.fit(X_raw, y_raw)
w = clf.coef_.ravel()
b = clf.intercept_
clf.score(X_raw, y_raw)
plot_decision_boundary(w, b, X_raw, 1000)
plot_confidence(clf, X_raw)<jupyter_output><empty_output><jupyter_text>2 SVMs with gaussian kernel<jupyter_code>data = loadmat('data/ex6data2.mat')
X_raw = data['X']
y_raw = data['y'].ravel()

X_positive = X_raw[y_raw == 1]
X_negative = X_raw[y_raw == 0]
print(X_positive.shape)
print(X_negative.shape)
plot_raw(X_positive, X_negative)<jupyter_output><empty_output><jupyter_text>Gaussian kernel<jupyter_code>def gaussian_kernel(x1, x2, sigma):
    return np.exp(-((x1 - x2) ** 2).sum() / (2 * sigma ** 2))
x1 = np.array([1.0, 2.0, 1.0])
x2 = np.array([0.0, 4.0, -1.0])
gaussian_kernel(x1, x2, 2)<jupyter_output><empty_output><jupyter_text>Try sklearn built-in gaussian kernel<jupyter_code>clf = svm.SVC(C=100, kernel='rbf', gamma=10, probability=True)
clf.fit(X_raw, y_raw)
clf.score(X_raw, y_raw)
prob = clf.predict_proba(X_raw)[:, 0]

fig, ax = plt.subplots(figsize=(12, 8))
ax.scatter(X_raw[:, 0], X_raw[:, 1], s=30, c=prob, cmap='Reds')<jupyter_output><empty_output><jupyter_text>Select best parameters with cross validation set<jupyter_code>data = loadmat('data/ex6data3.mat')
X_train = data['X']
y_train = data['y'].ravel()
X_cv = data['Xval']
y_cv = data['yval'].ravel()

X_positive = X_train[y_train == 1]
X_negative = X_train[y_train == 0]
print(X_positive.shape)
print(X_negative.shape)
plot_raw(X_positive, X_negative)
def select_param(X_train, y_train, X_cv, y_cv, Cs, sigmas):
    best_score = 0
    best_param = {'C': None, 'sigma': None}
    
    for C in Cs:
        for sigma in sigmas:
            clf = svm.SVC(C=C, kernel='rbf', gamma=sigma)
            clf.fit(X_train, y_train)
            score = clf.score(X_cv, y_cv)
            
            if score > best_score:
                best_score = score
                best_param['C'] = C
                best_param['sigma'] = sigma
                
    return best_score, best_param
options = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]
score, param = select_param(X_train, y_train, X_cv, y_cv, Cs=options, sigmas=options)
print(score)
print(param)
clf = svm.SVC(C=3, kernel='rbf', gamma=30, probability=True)
clf.fit(X_train, y_train)
prob = clf.predict_proba(X_train)[:, 0]

fig, ax = plt.subplots(figsize=(12, 8))
ax.scatter(X_train[:, 0], X_train[:, 1], s=30, c=prob, cmap='Reds')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17139001349527666, 'fraction_numerical': 0.03396311291048133, 'mean_word_length': 4.859025032938076, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '394763', 'license': 'MIT', 'n_tokens_mistral': 1975, 'n_tokens_neox': 1849, 'n_words': 351, 'path': 'exercise/ex6_svm/1_svm.ipynb', 'repo_name': 'Zx55/Coursera-AndrewNg-ML', 'size': '4446'}