<jupyter_start><jupyter_text>Linear SVM Classification<jupyter_code>def plot_svc_decision_boundary(svm_clf, xmin, xmax):
    """plot decision boundary for Linear SVM  
    
    :param svm_clf: svm model
    :param xmin: the lower limit of the x column 
    :param xmax: the upper limit of the x column
    """
    
    w = svm_clf.coef_[0]
    b = svm_clf.intercept_[0]

    # At the decision boundary, w0*x0 + w1*x1 + b = 0
    # => x1 = -w0/w1 * x0 - b/w1
    x0 = np.linspace(xmin, xmax, 200)
    decision_boundary = -w[0]/w[1] * x0 - b/w[1]

    margin = 1/w[1]
    gutter_up = decision_boundary + margin
    gutter_down = decision_boundary - margin

    svs = svm_clf.support_vectors_
    plt.scatter(svs[:, 0], svs[:, 1], s=10, facecolors='#FFAAAA')
    plt.plot(x0, decision_boundary, "k-", linewidth=2)
    plt.plot(x0, gutter_up, "k--", linewidth=2)
    plt.plot(x0, gutter_down, "k--", linewidth=2)
from sklearn.svm import SVC

# SVM Classifier model, a smaller C value leads to a wider street but more margin violations
svm_clf = SVC(kernel="linear", C=1000)
svm_clf.fit(X_reduced, y)
plot_dataset(X_reduced, y, axes_range)
plot_svc_decision_boundary(svm_clf, -4, 4)<jupyter_output><empty_output><jupyter_text>Non-Linear SVM Classification<jupyter_code>def plot_predictions(clf, axes):
    """plot the non-linear classification plot
    
    :param clf: the classifier
    :param axes: list, the range of the axes respectively
    """
    
    x0s = np.linspace(axes[0], axes[1], 100)
    x1s = np.linspace(axes[2], axes[3], 100)
    x0, x1 = np.meshgrid(x0s, x1s)
    X = np.c_[x0.ravel(), x1.ravel()]
    y_pred = clf.predict(X).reshape(x0.shape)
    # calculate the distance of the samples X to the separating hyperplane
    y_prediction = clf.decision_function(X).reshape(x0.shape)
    plt.contourf(x0, x1, y_pred, cmam=plt.cm.brg, alpha=0.2)
    plt.contourf(x0, x1, y_prediction, cmap=plt.cm.brg, alpha=0.1)
from sklearn.preprocessing import PolynomialFeatures
from sklearn.svm import LinearSVC

poly_kernel_svm_clf = SVC(kernel="poly", degree=3, coef0=1, C=5)
poly_kernel_svm_clf.fit(X_reduced, y)
plot_dataset(X_reduced, y, axes_range)
plot_predictions(poly_kernel_svm_clf, axes_range)

save_fig("kernelized_polynomial_svc_plot")
rbf_kernel_svm_clf = SVC(kernel="rbf", gamma=0.1, C=10)
rbf_kernel_svm_clf.fit(X_reduced, y)
sigmoid_kernel_svm_clf = SVC(kernel="sigmoid", gamma=0.1, coef0=1, C=10)
sigmoid_kernel_svm_clf.fit(X_reduced, y)
fig, axes = plt.subplots(ncols=2, figsize=(10.5, 4), sharey=True)

plt.sca(axes[0])
plot_dataset(X_reduced, y, axes_range)
plot_predictions(rbf_kernel_svm_clf, axes_range)
plt.title(r"$\gamma=0.1, C=10$")

plt.sca(axes[1])
plot_dataset(X_reduced, y, axes_range)
plot_predictions(sigmoid_kernel_svm_clf, axes_range)
plt.title(r"$\gamma=0.1, coef0=1, C=10$")

plt.show()
import time

# tolerance for stopping criterion
tol = 0.1
tols = []
times = []
for i in range(10):
    svm_clf1 = SVC(kernel="poly", gamma=3, C=10, tol=tol, verbose=1)
    t1 = time.time()
    svm_clf1.fit(X_reduced, y)
    t2 = time.time()
    times.append(t2-t1)
    tols.append(tol)
    print(i, tol, t2-t1)
    tol /= 10
# make a plot with log scaling on the x axis
plt.semilogx(tols, times, "bo-")
plt.xlabel("Tolerance", fontsize=16)
plt.ylabel("Time (seconds)", fontsize=16)
plt.grid(True)
plt.show()
save_fig("tolerance_time_plot")<jupyter_output>[LibSVM]0 0.1 0.31252408027648926
[LibSVM]1 0.01 0.5576381683349609
[LibSVM]2 0.001 2.8446741104125977
[LibSVM]3 0.0001 3.0710320472717285
[LibSVM]4 1e-05 3.4556779861450195
[LibSVM]5 1.0000000000000002e-06 3.5690419673919678
[LibSVM]6 1.0000000000000002e-07 2.694470167160034
[LibSVM]7 1.0000000000000002e-08 3.448937177658081
[LibSVM]8 1.0000000000000003e-09 3.4144837856292725
[LibSVM]9 1.0000000000000003e-10 3.493892192840576<jupyter_text>Grid Search<jupyter_code># data sets that don't contain classification label
X = df.drop("TRUE VALUE", axis=1)
from sklearn.model_selection import train_test_split

# split data set into training set and testing set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
from sklearn.preprocessing import StandardScaler

# svm algorithm is sensitive to scaling
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
svm_clf2 = SVC(kernel="rbf", gamma="scale")
svm_clf2.fit(X_train_scaled, y_train)
from sklearn.metrics import accuracy_score

y_pred = svm_clf2.predict(X_train_scaled)
accuracy_score(y_train, y_pred)
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import reciprocal, uniform

# find the best combination of hyperparameters of SVM model
param_distributions = {"gamma": reciprocal(0.001, 0.1), "C": uniform(1, 10)}
rnd_search_cv = RandomizedSearchCV(svm_clf2, param_distributions, n_iter=10, verbose=2, cv=3)
rnd_search_cv.fit(X_train_scaled, y_train)
# the best model among ten of the trained models 
rnd_search_cv.best_estimator_
# the best score of the best model
rnd_search_cv.best_score_<jupyter_output><empty_output><jupyter_text>Metrics for Binary Classifiers<jupyter_code>from sklearn.model_selection import cross_val_predict

svm_clf3 = SVC(kernel="rbf", gamma="scale")

# get a "clean" prediction for each instance in the training set
y_train_pred = cross_val_predict(svm_clf3, X_train_scaled, y_train, cv=3)
y_train_pred<jupyter_output><empty_output><jupyter_text>Precision & Recall<jupyter_code>from sklearn.metrics import confusion_matrix

# confusion matrix, each row represents an actual class, while each column represented a predicted class
cm = confusion_matrix(y_train, y_train_pred)
cm
import seaborn as sn

classes = list(y_train.unique())
# location for labels in x-axis
tick_marks = np.arange(len(classes)) + 0.5
plt.title("Confusion Matrix")
sn.heatmap(cm, annot=True, cmap="BuPu")
plt.xticks(tick_marks, classes)
plt.yticks(tick_marks, classes)
from sklearn.metrics import precision_score, recall_score

# The precision is intuitively the ability of the classifier not to label a sample, which is negative, as positive.
precision_score(y_train, y_train_pred)
# The recall is intuitively the ability of the classifier to find all the positive samples.
recall_score(y_train, y_train_pred)
from sklearn.metrics import f1_score

# F1 = 2 * (precision * recall) / (precision + recall)
f1_score(y_train, y_train_pred)
from sklearn.metrics import classification_report

# build a text report showing the main classification metrics.
print(classification_report(y_train, y_train_pred))
# each element represents whether a predicted sample for x_train (seen as testing set in cross_val_predict) 
# by the classifier lies to the right or left side of the Hyperplane and also how far from the HyperPlane.
y_scores = cross_val_predict(svm_clf3, X_train_scaled, y_train, cv=3,
                             method="decision_function")
# how confidently each value predicted for x_test by the classifier 
# is Positive ( large-magnitude Positive value ) or Negative ( large-magnitude Negative value)
y_scores
from sklearn.metrics import precision_recall_curve

# compute precision-recall pairs for different probability thresholds
precisions, recalls, thresholds = precision_recall_curve(y_train, y_scores)
precision_recall_curve?
def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    """plot precision_recall_curve
    
    :param precisions: array
    :param recalls: array
    :param thresholds: array, increasing thresholds on the decision function used to compute precision and recall.
    """
    
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision", linewidth=2)
    plt.plot(thresholds, recalls[:-1], "g-", label="Recall", linewidth=2)
    plt.legend(loc="center right", fontsize=16) 
    plt.xlabel("Threshold", fontsize=16)       
    plt.grid(True)                                       
plt.figure(figsize=(8, 4))                                                                  
plot_precision_recall_vs_threshold(precisions, recalls, thresholds)
save_fig("precision_recall_vs_threshold_plot")                                              
plt.show()
def plot_precision_vs_recall(precisions, recalls):
    """plot precision_vs_recall
    
    :param precisions: array
    :param recalls: array
    
    """
    
    plt.plot(recalls, precisions, "b-", linewidth=2)
    plt.xlabel("Recall", fontsize=16)
    plt.ylabel("Precision", fontsize=16)
    plt.axis([0, 1, 0, 1])
    plt.grid(True)
    
plt.figure(figsize=(8, 6))
plot_precision_vs_recall(precisions, recalls)
save_fig("precision_vs_recall_plot")
plt.show()
# np.argmax() will find the first index of the maximum value, which in this case means the first True value
threshold_90_precision = thresholds[np.argmax(precisions >= 0.90)]
threshold_90_precision
# if we want a 90% precision classifier, set the element as True if its decision_function value beyond the 90% precision threshold
y_train_pred_90 = (y_scores >= threshold_90_precision)
y_train_pred_90
y_train_true = []
# alternate 1 with True, -1 with False
for i in range(len(y_train)):
    if list(y_train)[i] == 1:
        y_train_true.append(True)
    else:
        y_train_true.append(False)
# the 90% precision classifier
precision_score(y_train_true, y_train_pred_90)
# the corresponding recall value in terms of the 90% precision classifier
recall_score(y_train_true, y_train_pred_90)<jupyter_output><empty_output><jupyter_text>ROC Curves<jupyter_code>from sklearn.metrics import roc_curve

# compute Receiver operating characteristic (ROC)
fpr, tpr, thresholds = roc_curve(y_train, y_scores)
def plot_roc_curve(fpr, tpr):
    """plots the true positive rate (another name for recall) against the false positive rate
    
    :param fpr: false positive rate
    :param tpr: true positive rate
    """
    
    plt.plot(fpr, tpr, linewidth=2)
    plt.plot([0, 1], [0, 1], 'k--') # dashed diagonal
    plt.axis([0, 1, 0, 1])                                    
    plt.xlabel('False Positive Rate (Fall-Out)', fontsize=16) 
    plt.ylabel('True Positive Rate (Recall)', fontsize=16)    
    plt.grid(True)   

plt.figure(figsize=(8, 6))  
# The dashed diagonal line represents the ROC curve of a purely random classifier;
# a good classifier stays as far away from that line as possible (toward the top-left corner)
plot_roc_curve(fpr, tpr)
save_fig("roc_curve_plot")                        
plt.show()
from sklearn.metrics import roc_auc_score

# the area under the curve, perfect classifier will have a ROC AUC equal to 1,
# whereas a purely random classifier will have a ROC AUC equal to 0.5
roc_auc_score(y_train, y_scores)<jupyter_output><empty_output><jupyter_text>Multiclass Classification<jupyter_code>df2 = pd.read_excel("data2.xlsx", engine="openpyxl")
df2.head()
df2.info()
X = df2.drop("TRUE VALUE", axis=1)
y = df2["TRUE VALUE"]
y.unique()
y.value_counts()
from sklearn.model_selection import train_test_split

# split data set into training set and testing set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
from sklearn.multiclass import OneVsOneClassifier

# one-versus-one strategy, train a series of binary classifier for every pair of classes
# e.g -1 vs 1, -1 vs 0, 0 vs 1
ovo_clf = OneVsOneClassifier(SVC(gamma="auto", random_state=42))
ovo_clf.fit(X_train, y_train)
y_pred_ovo = ovo_clf.predict(X_test)
from sklearn.metrics import accuracy_score
accuracy_score(y_test, y_pred_ovo)
# if there are N classes, it will train N X (N - 1) / 2 classifiers
len(ovo_clf.estimators_)
from sklearn.multiclass import OneVsRestClassifier

# this strategy consists in fitting one classifier per class.
# for each classifier, the class is fitted against all the other classes
# e.g -1 vs non -1, 1 vs non 1, 0 vs non 0
ovr_clf = OneVsRestClassifier(SVC(gamma="auto", random_state=42))
ovr_clf.fit(X_train, y_train)
y_pred_ovr = ovr_clf.predict(X_test)
from sklearn.metrics import accuracy_score
accuracy_score(y_test, y_pred_ovr)
# if there are N classes, it will train N classifiers
len(ovr_clf.estimators_)
from sklearn.model_selection import cross_val_score

# use cross-validation skill
ovo_svm_scores = cross_val_score(ovo_clf, X_train, y_train, cv=10)
ovo_svm_scores.mean()
# use cross-validation skill
ovr_svm_scores = cross_val_score(ovr_clf, X_train, y_train, cv=10)
ovr_svm_scores.mean()
plt.figure(figsize=(8, 4))
plt.plot([1]*10, ovo_svm_scores, ".")
plt.plot([2]*10, ovr_svm_scores, ".")
# if the lower quartile is Q1 and the upper quartile is Q3,
# then the interquartile range IQR = Q3 - Q1 (this is the box's height),
# and any score lower than Q1 - 1.5 x IQR is a flier("outliers"),
# and so is any score greater than Q3 + 1.5 X IQR 
plt.boxplot([ovo_svm_scores, ovr_svm_scores], labels=("OVO_SVM","OVR_SVM"))
plt.ylabel("Accuracy", fontsize=14)
plt.show()<jupyter_output><empty_output><jupyter_text>Multilabel Classification<jupyter_code>from sklearn.neighbors import KNeighborsClassifier

# classes like 0, 1
y_train_non_neg = (y_train >= 0)
# class like -1
y_train_neg = (y_train < 0)
y_multilabel = np.c_[y_train_non_neg, y_train_neg]

# KNN supports multilabel classification, but not all classifiers (i.e, SVC) do
knn_clf = KNeighborsClassifier()
knn_clf.fit(X_train, y_multilabel)
knn_clf.predict(X_test)
from sklearn.metrics import f1_score

# F1 score for each individual label, then simply compute the average score
y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3)
f1_score(y_multilabel, y_train_knn_pred, average="macro")
# give weight to the classifier's score on the label
# depending on the number of instances with that target label
f1_score(y_multilabel, y_train_knn_pred, average="weighted")<jupyter_output><empty_output><jupyter_text>Error Analysis<jupyter_code>from sklearn.metrics import plot_confusion_matrix

svm_clf4 = SVC(gamma="scale")
svm_clf4.fit(X_train, y_train)
plot_confusion_matrix(svm_clf4, X_train, y_train)
from sklearn.model_selection import cross_val_predict

svm_clf5 = SVC(gamma="scale")
y_train_pred = cross_val_predict(svm_clf5, X_train, y_train, cv=3)
conf_mx = confusion_matrix(y_train, y_train_pred)
conf_mx
def confusion_matrix_plot(conf_mx, classes):
    """plot confusion matrix
    
    :param conf_mx: confusion matrix value
    :param classes: the name of the classes
    """
    
    # location for labels in x-axis
    tick_marks = np.arange(len(classes))
    plt.matshow(conf_mx, cmap=plt.cm.gray)
    plt.title("Confusion Matrix")
    plt.xticks(tick_marks, classes)
    plt.yticks(tick_marks, classes)
confusion_matrix_plot(conf_mx, [-1, 0, 1])
# divide each value in the confusion matrix by the number of true corresponding classes,
# compare error rate instead of absolute number of errors, which would make abundant classes look unfairly bad
row_sums = conf_mx.sum(axis=1, keepdims=True)
norm_conf_mx = conf_mx / row_sums
np.fill_diagonal(norm_conf_mx, 0)
# the column for class 1 is quite bright, which tells many elements misclassified as 1
confusion_matrix_plot(norm_conf_mx, [-1, 0, 1])<jupyter_output><empty_output><jupyter_text>SVM Regression<jupyter_code>df3 = pd.read_excel("data3.xlsx", engine="openpyxl")
df3.head()
df3.info()
X = df3.drop("Fe3+/Fetot", axis=1)
y = df3["Fe3+/Fetot"]
from sklearn.model_selection import train_test_split

# split data set into training set and testing set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
def find_support_vectors(svm_reg, X, y):
    """find support vectors of SVR
    
    :param svm_reg: a SVM Regressor
    :param X: data
    :param y: labels
    """
    y_pred = svm_reg.predict(X)
    # evalute whether the instance is off the street line
    off_margin = np.array(np.abs(y - y_pred) >= svm_reg.epsilon)
    return np.argwhere(off_margin)
def plot_svm_regression(svm_reg, X, y, axes):

    x1s = np.linspace(axes[0], axes[1], 100).reshape(100, 1)
    y_pred = svm_reg.predict(x1s)
    plt.plot(x1s, y_pred, "k-", linewidth=2, label=r"$\hat{y}$")
    plt.plot(x1s, y_pred + svm_reg.epsilon, "k--")
    plt.plot(x1s, y_pred - svm_reg.epsilon, "k--")
    plt.scatter(X[svm_reg.support_].reshape(-1, 1), np.array(y)[svm_reg.support_].reshape(-1, 1), s=180, facecolors='#FFAAAA')
    
    plt.plot(X, y, "bo")
    plt.xlabel(r"$x_1$", fontsize=18)
    plt.legend(loc="upper left", fontsize=18)
    plt.axis(axes)
from sklearn.decomposition import PCA

# PCA assumes that the origin is centered, the data set is centralized automatically
pca = PCA(n_components=1)
# reduce the data set down to 2-dimensional space
X_reduced = pca.fit_transform(X)
X_reduced.shape
from sklearn.svm import LinearSVR

svm_reg1 = LinearSVR(epsilon=0.1, random_state=42)
svm_reg2 = LinearSVR(epsilon=0.2, random_state=42)
svm_reg1.fit(X_reduced, y)
svm_reg2.fit(X_reduced, y)
svm_reg1.support_ = find_support_vectors(svm_reg1, X_reduced, y)
svm_reg2.support_ = find_support_vectors(svm_reg2, X_reduced, y)
X_reduced[svm_reg1.support_].reshape(-1, 1).shape
axes_range = [X_reduced.min(), X_reduced.max(), y.min(), y.max()]
axes_range
import matplotlib.pyplot as plt

fig, axes = plt.subplots(ncols=2, figsize=(9, 4), sharey=True)
plt.sca(axes[0])
plot_svm_regression(svm_reg1, X_reduced, y, axes_range)
plt.title(r"$\epsilon = {}$".format(svm_reg1.epsilon), fontsize=18)
plt.ylabel(r"$y$", fontsize=18, rotation=0)

plt.sca(axes[1])
plot_svm_regression(svm_reg2, X_reduced, y, axes_range)
plt.title(r"$\epsilon = {}$".format(svm_reg2.epsilon), fontsize=18)
save_fig("svm_regression_plot")
plt.show()
from sklearn.svm import SVR

svm_poly_reg1 = SVR(kernel="poly", degree=2, C=100, epsilon=0.1, gamma="scale")
svm_poly_reg2 = SVR(kernel="poly", degree=2, C=0.01, epsilon=0.1, gamma="scale")
svm_poly_reg1.fit(X_reduced, y)
svm_poly_reg2.fit(X_reduced, y)
fig, axes = plt.subplots(ncols=2, figsize=(9, 4), sharey=True)
plt.sca(axes[0])
plot_svm_regression(svm_poly_reg1, X_reduced, y, axes_range)
plt.title(r"$degree={}, C={}, \epsilon = {}$".format(svm_poly_reg1.degree, svm_poly_reg1.C, svm_poly_reg1.epsilon), fontsize=18)
plt.ylabel(r"$y$", fontsize=18, rotation=0)
plt.sca(axes[1])
plot_svm_regression(svm_poly_reg2, X_reduced, y, axes_range)
plt.title(r"$degree={}, C={}, \epsilon = {}$".format(svm_poly_reg2.degree, svm_poly_reg2.C, svm_poly_reg2.epsilon), fontsize=18)
save_fig("svm_with_polynomial_kernel_plot")
plt.show()<jupyter_output>Saving figure svm_with_polynomial_kernel_plot
{'fraction_non_alphanumeric': 0.11173123814684367, 'fraction_numerical': 0.037984286101327557, 'mean_word_length': 5.143808255659121, 'pattern_counts': {'":': 2, '<': 40, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '133964', 'license': 'MIT', 'n_tokens_mistral': 7409, 'n_tokens_neox': 6786, 'n_words': 1646, 'path': 'english/svm/svm.ipynb', 'repo_name': 'ZJUEarthData/new-algorithm', 'size': '18455'}