<jupyter_start><jupyter_text>Support Vector Classifier<jupyter_code>import pandas as pd
import seaborn as sns
import numpy as np
import download_data as dl
import matplotlib.pyplot as plt
import sklearn.svm as svm
from sklearn import metrics
from conf_matrix import func_confusion_matrix
from sklearn.cross_validation import cross_val_score

%matplotlib inline


## step 1: load data from csv file. 
data = dl.download_data('crab.csv').values

n = 200
#split data 
S = np.random.permutation(n)
#100 training samples
Xtr = data[S[:100], :6]
Ytr = data[S[:100], 6:]
# 100 testing samples
X_test = data[S[100:], :6]
Y_test = data[S[100:], 6:].ravel()

## step 2 randomly split Xtr/Ytr into two even subsets: use one for training, another for validation.

n2 = len(Xtr)
S2 = np.random.permutation(n2)

x_train = Xtr[S2[:50], :6]
y_train = Ytr[S2[:50], :6]

x_validation = Xtr[S2[:50], :6]
y_validation = Ytr[S2[:50], :6].ravel()

## step 3 Model selection over validation set
# consider the parameters C, kernel types (linear, RBF etc.) and kernal
# parameters if applicable. 


# 3.1 Plot the validation errors while using different values of C ( with other hyperparameters fixed) 
#  keeping kernel = "linear"

c_range =  list(range(1,26))
svm_c_error = []
acc_score = []
for c_value in c_range:
    model = svm.SVC(kernel='linear', C=c_value)
    model.fit(X=x_train, y=y_train)
    error = 1. - model.score(x_validation, y_validation)
    svm_c_error.append(error)
    scores = cross_val_score(model, x_validation, y_validation, cv=10, scoring='accuracy')
    acc_score.append(scores.mean())
plt.plot(c_range, svm_c_error)
plt.title('Linear SVM')
plt.xlabel('c values')
plt.ylabel('error')
plt.xticks(c_range)
plt.show()

plt.plot(c_range, acc_score)
plt.title('Linear SVM')
plt.xlabel('c values')
plt.ylabel('accuracy')
plt.xticks(c_range)
plt.show()

y_pred=model.predict(X_test)

conf_matrix, accuracy, recall_array, precision_array = func_confusion_matrix(Y_test, y_pred)
print("Confusion Matrix: ")
print(conf_matrix)
print("Average Accuracy: {}".format(accuracy))
print("Per-Class Precision: {}".format(precision_array))
print("Per-Class Recall: {}".format(recall_array))<jupyter_output>/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was e[...]<jupyter_text>In evaluating the results of using the basic linear kernel over a range of C values, the average accuracy was surprisingly good, scoring a 0.92. The validation error score remains high until around the 7th C value, where it drops to near 0. With accuracy, the linear model maintains its highest accuracy in the C value range of 2 to 5. The best accuracy/error parameters seem to lie at the C value of 7 or 8.<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline

degree=[2,3,4,5,6]
svm_d_error = []
acc_score2 = []
for d in degree:
    model2 = svm.SVC(kernel='poly', degree=d)
    model2.fit(X=x_train, y=y_train)
    error = 1. - model2.score(x_validation, y_validation)
    svm_d_error.append(error)
    scores = cross_val_score(model2, x_validation, y_validation, cv=10, scoring='accuracy')
    acc_score2.append(scores.mean())
plt.plot(degree, svm_d_error)
plt.title('Polynomial SVM')
plt.xlabel('degree')
plt.ylabel('error')
plt.xticks(degree)
plt.show()

plt.plot(degree, acc_score2)
plt.title('Polynomial SVM')
plt.xlabel('degree')
plt.ylabel('accuracy')
plt.xticks(degree)
plt.show()

y_pred=model2.predict(X_test)

conf_matrix, accuracy, recall_array, precision_array = func_confusion_matrix(Y_test, y_pred)
print("Confusion Matrix: ")
print(conf_matrix)
print("Average Accuracy: {}".format(accuracy))
print("Per-Class Precision: {}".format(precision_array))
print("Per-Class Recall: {}".format(recall_array))<jupyter_output>/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was e[...]<jupyter_text>For the polynomial SVM model, I evaluated the range of degrees from 2 to 6. The average error remained consistent at 0, while the accuracy score appeared to exponentially ascend and normalize at a degree of 4. The polynomial model (using fixed hyperparameters) performed nearly equivalent to the basic linear model results.<jupyter_code>gamma_range=[0.01,0.02,0.03,0.04,0.05]
acc_score3=[]
svm_g_error = []
for g in gamma_range:
    model3 = svm.SVC(kernel='rbf', gamma=g)
    model3.fit(X=x_train, y=y_train)
    error = 1. - model3.score(x_validation, y_validation)
    svm_g_error.append(error)
    scores = cross_val_score(model3, x_validation, y_validation, cv=10, scoring='accuracy')
    acc_score3.append(scores.mean())
    
plt.plot(degree, svm_g_error)
plt.title('RBF SVM')
plt.xlabel('gamma')
plt.ylabel('error')
plt.xticks(degree)
plt.show()

plt.plot(degree, acc_score3)
plt.title('RBF SVM')
plt.xlabel('gamma')
plt.ylabel('accuracy')
plt.xticks(degree)
plt.show()

y_pred=model3.predict(X_test)

conf_matrix, accuracy, recall_array, precision_array = func_confusion_matrix(Y_test, y_pred)
print("Confusion Matrix: ")
print(conf_matrix)
print("Average Accuracy: {}".format(accuracy))
print("Per-Class Precision: {}".format(precision_array))
print("Per-Class Recall: {}".format(recall_array))<jupyter_output>/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was e[...]<jupyter_text>In evaluating the RBF SVM model, I iterated through the gamma range of .01 to .05. This model scored the lowest in accuracy with a 0.82, appearing to optimize its paramters around a gamma value of .04.<jupyter_code>## step 4 Select the best model and apply it over the testing subset 
best_kernel_poly = 'poly'
best_d = 2 
best_model_poly = svm.SVC(kernel=best_kernel_poly, degree=best_d)
best_model_poly.fit(X=x_train, y=y_train)
## step 5 evaluate your results with the metrics you have developed in HA3,including accuracy, quantize your results. 
y_pred=best_model_poly.predict(X_test)

conf_matrix, accuracy, recall_array, precision_array = func_confusion_matrix(Y_test, y_pred)
print("Confusion Matrix: ")
print(conf_matrix)
print("Average Accuracy: {}".format(accuracy))
print("Per-Class Precision: {}".format(precision_array))
print("Per-Class Recall: {}".format(recall_array))
## step 4 Select the best model and apply it over the testing subset 

best_kernel_linear = 'linear'
best_c = 5 
best_model_linear = svm.SVC(kernel=best_kernel_linear, C=best_c)
best_model_linear.fit(X=x_train, y=y_train)
## step 5 evaluate your results with the metrics you have developed in HA3,including accuracy, quantize your results. 

y_pred=best_model_linear.predict(X_test)

conf_matrix, accuracy, recall_array, precision_array = func_confusion_matrix(Y_test, y_pred)
print("Confusion Matrix: ")
print(conf_matrix)
print("Average Accuracy: {}".format(accuracy))
print("Per-Class Precision: {}".format(precision_array))
print("Per-Class Recall: {}".format(recall_array))<jupyter_output>Confusion Matrix: 
[[43  5]
 [ 3 49]]
Average Accuracy: 0.92
Per-Class Precision: [ 0.93478261  0.90740741]
Per-Class Recall: [ 0.89583333  0.94230769]<jupyter_text>In determining the best model, I compared the two highest performing base models, polynomial and linear, and then tweaked hyperparamters for optimization. My results show that the simple linear model (using a C value of 5) achieved a 0.96 accuracy score, significantly beating the average accuracy scores of 0.92 for the optimized polynomial model.<jupyter_code>## step 6 Show 5 correct and 5 incorrect predictions by the best model

df = pd.DataFrame(X_test)
df["actual"] = Y_test
df["predicted"] = y_pred

correct = df[df["actual"] == df["predicted"]]
incorrect = df[df["actual"] != df["predicted"]]

# 5 correct prediction samples
correct.head(n=5)
# 5 incorrect prediction samples
incorrect.head(n=5)
# function for building weak classifiers, i.e.:  stump function

import numpy as py

def buildWeakStump(d,l,D): # (data, label, weight)
    dataMatrix = py.mat(d)
    labelmatrix = py.mat(l).T
    m,n = py.shape(dataMatrix)
    numstep = 10.0
    bestStump = {}
    bestClass = py.mat(py.zeros((5,1)))
    minErr = py.inf
    for i in range(n):
        datamin = dataMatrix[:,i].min()
        datamax = dataMatrix[:,i].max()
        stepSize = (datamax - datamin) / numstep
        for j in range(-1,int(numstep)+1):
            for inequal in ['lt','gt']:
                threshold = datamin + float(j) * stepSize
                predict = stumpClassify(dataMatrix,i,threshold,inequal)
                err = py.mat(py.ones((m,1)))
                err[predict == labelmatrix] = 0
                weighted_err = D.T * err;
                if weighted_err < minErr:
                    minErr = weighted_err
                    bestClass = predict.copy()
                    bestStump['dim'] = i
                    bestStump['threshold'] = threshold
                    bestStump['ineq'] = inequal
    return bestStump, minErr, bestClass

# Use a weak classifier, i.e. a decision stump, to classify data

def stumpClassify(datamat,dim,threshold,inequal):
    res = py.ones((py.shape(datamat)[0],1))
    if inequal == 'lt':
        res[datamat[:,dim] <= threshold] = -1.0
    else:
        res[datamat[:,dim] > threshold] = -1.0
    return res

# Boosting Algorithm

def train(data,label,numIt = 1000):
    
    weakClassifiers = []
    #m is the number of samples
    m = py.shape(data)[0]
    # sample weights, 1/m at the beginning
    D = py.mat(py.ones((m,1))/m) 
    
    estStrong = py.mat(py.zeros((m,1)))
    for i in range(numIt):
        # bestStump: weak classifier; error: error rate
        bestStump, error, classEstimate = buildWeakStump(data,label,D)
        print("D:",D.T)
        
        # calculate the weight of the selected decision stump based on its error rate
        alpha = float(0.5*py.log((1.0-error)/max(error,1e-16)))
        
        # add one more field to bestStump, i.e. classifier weight
        bestStump['alpha'] = alpha
        # add bestStump to the list of weak classifiers
        weakClassifiers.append(bestStump)
        print("classEstimate: ",classEstimate.T)

        #calculate sample weights (of all samples) 
        # set sample weights
        expon = py.multiply(-1*alpha*py.mat(label).T,classEstimate)
        D = py.multiply(D,py.exp(expon))
        # normalize D
        D = D/D.sum()
        
        
        estStrong += classEstimate*alpha
        
        EnsembleErrors = py.multiply(py.sign(estStrong)!=py.mat(label).T,\
                                  py.ones((m,1)))  #Converte to float
        
        errorRate = EnsembleErrors.sum()/m
        
        print("current error:  ",errorRate)
        if errorRate == 0.0:
            break
    return weakClassifiers

# Applying an adaboost classifier for a single data sample

def adaboostClassify(dataTest,classifier):
    dataMatrix = py.mat(dataTest)
    m = py.shape(dataMatrix)[0]
    estStrong = py.mat(py.zeros((m,1)))
    for i in range(len(classifier)):
        # call the function stumpClassify()
        classEstimate = stumpClassify(dataMatrix, 
                                      classifier[i]['dim'], 
                                      classifier[i]['threshold'], 
                                      classifier[i]['ineq'])
        # accumulate all predictions
        estStrong += classifier[i]['alpha']*classEstimate
    return py.sign(estStrong)

# Applying an adaboost classifier for all testing samples
def test(dataSet,classifier):
    label = []
    for i in range(py.shape(dataSet)[0]):
        label.append(adaboostClassify(dataSet[i,:],classifier))
    return label



#training
classifier = train(df.values[:, :6],df.values[:, 6],150)
print('done training\n')
#testing
test(X_test,classifier)
print('done teting\n')
import numpy as np
import matplotlib.pyplot as plt
import sklearn
from sklearn import datasets
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import zero_one_loss
from sklearn.ensemble import AdaBoostClassifier


n_estimators = 400
# A learning rate of 1. may not be optimal for both SAMME and SAMME.R
learning_rate = 1.

dt_stump = DecisionTreeClassifier(max_depth=1, min_samples_leaf=1)
dt_stump.fit(x_train, y_train)
dt_stump_err = 1.0 - dt_stump.score(X_test, Y_test)

dt = DecisionTreeClassifier(max_depth=9, min_samples_leaf=1)
dt.fit(x_train, y_train)
dt_err = 1.0 - dt.score(X_test, Y_test)


ada_discrete = AdaBoostClassifier(
    base_estimator=dt_stump,
    learning_rate=learning_rate,
    n_estimators=n_estimators,
    algorithm="SAMME")
ada_discrete.fit(x_train, y_train)

ada_real = AdaBoostClassifier(
    base_estimator=dt_stump,
    learning_rate=learning_rate,
    n_estimators=n_estimators,
    algorithm="SAMME.R")
ada_real.fit(x_train, y_train)

fig = plt.figure()
ax = fig.add_subplot(111)

ada_discrete_err = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_discrete.staged_predict(X_test)):
    ada_discrete_err[i] = zero_one_loss(y_pred, Y_test)

ada_discrete_err_train = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_discrete.staged_predict(x_validation)):
    ada_discrete_err_train[i] = zero_one_loss(y_pred, y_validation)

ada_real_err = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_real.staged_predict(X_test)):
    ada_real_err[i] = zero_one_loss(y_pred, Y_test)

ada_real_err_train = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_real.staged_predict(x_validation)):
    ada_real_err_train[i] = zero_one_loss(y_pred, y_validation)

ax.plot(np.arange(n_estimators) + 1, ada_discrete_err,
        label='Discrete AdaBoost Test Error',
        color='red')
ax.plot(np.arange(n_estimators) + 1, ada_discrete_err_train,
        label='Discrete AdaBoost Train Error',
        color='blue')
ax.plot(np.arange(n_estimators) + 1, ada_real_err,
        label='Real AdaBoost Test Error',
        color='orange')
ax.plot(np.arange(n_estimators) + 1, ada_real_err_train,
        label='Real AdaBoost Train Error',
        color='green')

ax.set_ylim((0.0, 0.5))
ax.set_xlabel('n_estimators')
ax.set_ylabel('error rate')

leg = ax.legend(loc='upper right', fancybox=True)
leg.get_frame().set_alpha(0.7)

plt.show()<jupyter_output>/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/Users/tylerkistler/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
{'fraction_non_alphanumeric': 0.11632183908045977, 'fraction_numerical': 0.020057471264367816, 'mean_word_length': 4.289057750759879, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '86858', 'license': 'MIT', 'n_tokens_mistral': 6191, 'n_tokens_neox': 5852, 'n_words': 1504, 'path': 'Adaboost Bagging/classifier_svm_adaboost.ipynb', 'repo_name': 'herr-kistler/machine-learning', 'size': '17400'}