<jupyter_start><jupyter_text>Preprocessing<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
np.random.seed(1)
X = np.random.normal(size=(20,2))
X
y_1 = np.array([1]*10)
y_minus1 = np.array([-1]*10)
y = np.concatenate([y_1, y_minus1])
y
X[y==1] += 1
X
plt.xkcd()
plt.figure(figsize=(25, 10))
plt.scatter(X[:,0][:],X[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('are the two classes linearly separable?', color='m', fontsize=30)
plt.xlabel('X[:, 0]', color='green', fontsize=20)
plt.ylabel('X[:, 1]', color='orange', fontsize=20)<jupyter_output><empty_output><jupyter_text>**Therefore, the two classes are not linearly separable.** **Support vector classifier**<jupyter_code>from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix, classification_report
svmfit10 = SVC(kernel='linear', C=10).fit(X, y)
def svmplot(svc, X, y, height=0.02, buffer=0.25):
    x_min, x_max = X[:, 0].min()-buffer, X[:, 0].max()+buffer
    y_min, y_max = X[:, 1].min()-buffer, X[:, 1].max()+buffer
    xx, yy = np.meshgrid(np.arange(x_min, x_max, height), np.arange(y_min, y_max, height))
    Z = svc.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.2)
    
plt.xkcd()
plt.figure(figsize=(25, 10))
svmplot(svmfit10, X, y)
plt.scatter(X[:,0][:],X[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('support vector classifier', color='m', fontsize=30)
plt.xlabel('X[:, 0]', color='green', fontsize=20)
plt.ylabel('X[:, 1]', color='orange', fontsize=20)
svmfit10.support_ # these are the support vectors
conf_mat10 = pd.DataFrame(confusion_matrix(y, svmfit10.predict(X)).T, index = svmfit.classes_, columns = svmfit.classes_)
conf_mat10
class_mat10 = classification_report(y, svmfit10.predict(X))
print(class_mat10)
svmfit001 = SVC(kernel='linear', C=0.001).fit(X, y) # using smaller value of cost vector

plt.figure(figsize=(25, 10))
svmplot(svmfit001, X, y)
plt.scatter(X[:,0][:],X[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('support vector classifier', color='m', fontsize=30)
plt.xlabel('X[:, 0]', color='green', fontsize=20)
plt.ylabel('X[:, 1]', color='orange', fontsize=20)
svmfit001.support_ # these are the support vectors<jupyter_output><empty_output><jupyter_text>*If we use smaller cost vector, there are a larger number of support vectors used because the margin is now wider.*<jupyter_code>conf_mat001 = pd.DataFrame(confusion_matrix(y, svmfit001.predict(X)).T, index = svmfit.classes_, columns = svmfit.classes_)
conf_mat001
class_mat001 = classification_report(y, svmfit001.predict(X))
print(class_mat001)<jupyter_output>precision    recall  f1-score   support

          -1       0.78      0.70      0.74        10
           1       0.73      0.80      0.76        10

   micro avg       0.75      0.75      0.75        20
   macro avg       0.75      0.75      0.75        20
weighted avg       0.75      0.75      0.75        20<jupyter_text>Using cross-validation to find the optimal cost<jupyter_code>from sklearn.model_selection import GridSearchCV as GSV
cost_range = [{'C': np.linspace(0.001, 20, 1000)}]
cost_cv= GSV(SVC(kernel='linear'), cost_range, cv=10, scoring='accuracy', return_train_score=True).fit(X, y)
best_params = cost_cv.best_params_
best_params['C']<jupyter_output><empty_output><jupyter_text>**GridSearchCV suggests that the best results are obtained at C=0.16115215215215214.**<jupyter_code>X_test = np.random.normal(size=(20,2))
X_test
y_1_test = np.array([1]*10)
y_minus1_test = np.array([-1]*10)
y_test = np.concatenate([y_1_test, y_minus1_test])
y
X_test[y_test==1] += 1
X_test
plt.xkcd()
plt.figure(figsize=(25, 10))
plt.scatter(X_test[:,0][:],X_test[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('are the two classes linearly separable?', color='m', fontsize=30)
plt.xlabel('X[:, 0]', color='green', fontsize=20)
plt.ylabel('X[:, 1]', color='orange', fontsize=20)
np.random.seed(101)
X_test = np.random.normal(size=(20,2))
y_test = np.random.choice([-1,1], 20)
X_test[y_test == 1] = X_test[y_test == 1]-1

plt.xkcd()
plt.figure(figsize=(25, 10))
plt.scatter(X_test[:,0][:],X_test[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('are the two classes linearly separable?', color='m', fontsize=30)
plt.xlabel('X_test[:, 0]', color='green', fontsize=20)
plt.ylabel('X_test[:, 1]', color='orange', fontsize=20)
svmfit_test = SVC(C=best_params['C'], kernel='linear').fit(X, y)

plt.figure(figsize=(25, 10))
svmplot(svmfit_test, X_test, y_test)
plt.scatter(X_test[:,0][:],X_test[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('support vector classifier', color='m', fontsize=30)
plt.xlabel('X_test[:, 0]', color='green', fontsize=20)
plt.ylabel('X_test[:, 1]', color='orange', fontsize=20)
conf_mat_test = pd.DataFrame(confusion_matrix(y_test, svmfit.predict(X_test)), index=svmfit_test.classes_, columns=svmfit_test.classes_)
conf_mat_test
class_mat = classification_report(y_test, svmfit.predict(X_test))
print(class_mat)
svmfit_test001 = SVC(C=10, kernel='linear')
svmfit_test001.fit(X, y)

plt.figure(figsize=(25, 10))
svmplot(svmfit_test, X_test, y_test)
plt.scatter(X_test[:,0][:],X_test[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('support vector classifier', color='m', fontsize=30)
plt.xlabel('X_test[:, 0]', color='green', fontsize=20)
plt.ylabel('X_test[:, 1]', color='orange', fontsize=20)
conf_mat_test = pd.DataFrame(confusion_matrix(y_test, svmfit_test001.predict(X_test)), index=svmfit_test001.classes_, columns=svmfit_test001.classes_)
conf_mat_test
class_mat_test001 = classification_report(y_test, svmfit_test.predict(X_test))
print(class_mat_test)<jupyter_output>precision    recall  f1-score   support

          -1       0.25      0.50      0.33         8
           1       0.00      0.00      0.00        12

   micro avg       0.20      0.20      0.20        20
   macro avg       0.12      0.25      0.17        20
weighted avg       0.10      0.20      0.13        20<jupyter_text>**Therefore, we see that C=10 provides worse results that C=0.16115215215215214. Using other values of C provide the same result.**  Support Vector classifier with linearly separable classes<jupyter_code>X = np.random.normal(size=(20,2))
X
y_1 = np.array([1]*10)
y_minus1 = np.array([-1]*10)
y = np.concatenate([y_1, y_minus1])
y
X[y==1] += 0.5
X
plt.xkcd()
plt.figure(figsize=(25, 10))
plt.scatter(X[:,0][:],X[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('are the two classes linearly separable?', color='m', fontsize=30)
plt.xlabel('X[:, 0]', color='green', fontsize=20)
plt.ylabel('X[:, 1]', color='orange', fontsize=20)
svmfit10 = SVC(kernel='linear', C=10).fit(X, y)
  
plt.xkcd()
plt.figure(figsize=(25, 10))
svmplot(svmfit10, X, y)
plt.scatter(X[:,0][:],X[:,1][:], marker='o', s=1000, c=['green', 'orange'])
plt.title('support vector classifier', color='m', fontsize=30)
plt.xlabel('X[:, 0]', color='green', fontsize=20)
plt.ylabel('X[:, 1]', color='orange', fontsize=20)
svmfit10.support_ # these are the support vectors
conf_mat10 = pd.DataFrame(confusion_matrix(y, svmfit10.predict(X)).T, index = svmfit.classes_, columns = svmfit.classes_)
conf_mat10
class_mat10 = classification_report(y, svmfit10.predict(X))
print(class_mat10)<jupyter_output>precision    recall  f1-score   support

          -1       0.69      0.90      0.78        10
           1       0.86      0.60      0.71        10

   micro avg       0.75      0.75      0.75        20
   macro avg       0.77      0.75      0.74        20
weighted avg       0.77      0.75      0.74        20
{'fraction_non_alphanumeric': 0.18089536699635606, 'fraction_numerical': 0.07131702238417491, 'mean_word_length': 4.675775480059084, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '189457', 'license': 'MIT', 'n_tokens_mistral': 3333, 'n_tokens_neox': 3002, 'n_words': 609, 'path': 'Chapter 9/R Lab/9.6.1 Support Vector Classifier.ipynb', 'repo_name': 'arpanganguli/ISLR-Python', 'size': '7684'}