<jupyter_start><jupyter_text>Polynomial Kernels Load training and testing data in pandas dataframes<jupyter_code>def load_data():
    column_names = ['digit', 'intensity', 'symmetry']
    sep = '\s+'
    features_train = pd.read_table('http://www.amlbook.com/data/zip/features.train', sep=sep, names=column_names)
    features_test = pd.read_table('http://www.amlbook.com/data/zip/features.test', sep=sep, names=column_names)
    return features_train, features_test
features_train, features_test = load_data()<jupyter_output><empty_output><jupyter_text>Set output labels for one-vs-all classifiers<jupyter_code>classifiers_one_vs_all = dict(
    {'zero_vs_all': 0,
     'one_vs_all': 1,
     'two_vs_all': 2,
     'three_vs_all': 3,
     'four_vs_all': 4,
     'five_vs_all': 5,
     'six_vs_all': 6,
     'seven_vs_all': 7,
     'eight_vs_all': 8,
     'nine_vs_all': 9
    }
)<jupyter_output><empty_output><jupyter_text>Create one-versus-all dataframe by adding outputs for the different classifiers to the training dataframe<jupyter_code>def create_one_vs_all_dataframe(df, classifiers):
    # add binary labels to dataframe
    one_vs_all = pd.DataFrame(df, copy=True)
    for class_label, digit in classifiers.items():
        labels = one_vs_all.loc[one_vs_all['digit'] == digit, 'digit']
        labels.loc[:] = 1.0
        one_vs_all[class_label] = labels
        
    one_vs_all.fillna(-1.0, inplace=True)
    return one_vs_all
features_train_one_vs_all = create_one_vs_all_dataframe(features_train, classifiers_one_vs_all)<jupyter_output><empty_output><jupyter_text>Get dataset (inputs / outputs) for a specific classifier<jupyter_code>def get_dataset(df, class_label):
    inputs = np.array(df.loc[:, ['intensity', 'symmetry']])
    outputs = np.array(df.loc[:, class_label])
    data = np.column_stack((inputs, outputs))
    return data<jupyter_output><empty_output><jupyter_text>Create Support Vector Machine (SVM) class - this is not used due to how slow quadratic programming package is - left here for illustration of the general structure of algorithm<jupyter_code>class SVM:
    """ Support Vector Machine (SVM) class. """
    
    def __init__(self, Q=2, rbf=False):
        """ Create a Support Vector Machine Algorithm (SVM). 
        Args:
        Q (int): Degree of the polynomial kernel (Q >= 0)
        rbf (bool): Use the RBF kernel instead of the polynomial kernel
        """
        self.Q = Q
        self.rbf = rbf
        
    def train(self, inputs, outputs, C=0.01, margin='hard'):    
        """Training using Support Vector Machine Algorithm (SVM).
        Args:
        inputs (np.ndarray): Input points.
        outputs (np.ndarray): Targets.
        C (float): Upper bound of Lagrange multiplier (C > 0.0)
        margin (string): Margin to apply, 'hard' or 'soft'
        Returns:
        Tuple(np.ndarray, int): Weights, number of support vectors
        """
        if margin not in ['hard','soft']:
            raise ValueError('Margin must be "hard" or "soft".')
        
        xn = inputs
        yn = outputs
        N = len(xn)
        
        mat = []
        for row_idx in range(0, N):
            for col_idx in range(0, N):
                if self.rbf:
                    kernel = self.kernel_rbf(xn[row_idx], xn[col_idx])
                else:
                    kernel = self.kernel_poly(xn[row_idx], xn[col_idx])
                val = yn[row_idx] * yn[col_idx] * kernel
                mat.append(val)
        mat = np.array(mat).reshape((N, N))
        
        # form matrices for quadratic programming solver
        dim = len(xn[0])
        P = matrix(mat, tc='d')
        q = matrix(-np.ones(N), tc='d')
        b = matrix(0.0, tc='d')
        A = matrix(yn, tc='d')
        A = A.trans()
        G = matrix(-np.identity(N), tc='d')
        if margin == 'hard':
            G = matrix(-np.identity(N), tc='d')
            h = matrix(np.zeros(N), tc='d')
        elif margin == 'soft':
            G_zero = -np.identity(N)
            h_zero = np.zeros(N)
            G_C = np.identity(N)
            h_C = C * np.ones(N)
              
            G = matrix(np.concatenate((G_zero, G_C)), tc='d')
            h = matrix(np.concatenate((h_zero, h_C)), tc='d')
        
        #print('P = ', P)
        #print('q = ', q)
        #print('G = ', G)
        #print('h = ', h)
        #print('A = ', A)
        #print('b = ', b)
                
        # call qp solver to compute weights
        solvers.options['show_progress'] = False # supress solver output
        
        sol = solvers.qp(P, q, G, h, A, b)
        alpha = np.array(list(sol['x']))
        #print('alpha = ', sol['x'])
        
        weights = np.zeros(dim)
        #sv_idxs = []
        sv = []
        sv_alphas = []
        sv_outputs = []
        for n in range(0, N):
            if margin == 'hard':
                weights += alpha[n] * yn[n] * xn[n]
                if alpha[n] > 1e-5: # => xn[n] is support vector
                    sv.append(xn[n])
            elif margin == 'soft':
                #print('alpha[{0}] = {1}'.format(n, alpha[n]))
                if alpha[n] > 1e-5 and alpha[n] <= C: # => xn[n] is support vector
                    #sv_idxs.append(n)
                    sv.append(xn[n])
                    sv_alphas.append(alpha[n])
                    sv_outputs.append(yn[n])
                    
        # compute number of support vectors
        num_sv = len(sv)
        
        if (num_sv == 0):
            raise Exception('There are no support vectors.')
        
        bs = []
        for m in range(0, num_sv):
            if margin == 'hard':
                b = (1. / sv_outputs[m]) - np.dot(weights.T, sv[m])
            elif margin == 'soft':
                b = sv_outputs[m]
                for n in range(0, num_sv):
                    if self.rbf:
                        kernel = self.kernel_rbf(sv[n], sv[m])
                    else:
                        kernel = self.kernel_poly(sv[n], sv[m])
                    b -= sv_alphas[n] * sv_outputs[n] * kernel
            bs.append(b)
        bs_round = np.round(bs, 1)
        #print('bs sv = ', bs)
        
        #print('bs = ', bs)
        #print('bs_round = ', bs_round)
        #print('np.unique(bs_round) = ', np.unique(bs_round))
        
        #if (len(np.unique(bs_round)) != 1):
        #    raise Exception('All support vectors must produce the same value of b.')
            
        weights = np.insert(weights, 0, b)
        #print('weights = ', weights)

        if margin == 'hard':
            return weights, num_sv
        elif margin == 'soft':
            return np.array(sv_alphas), np.array(sv), np.array(sv_outputs), b
    
    def binary_error(self, sv_alphas, sv, sv_outputs, b, inputs, outputs):
        """ Evaluate binary classification error. 
        Args:
        sv_alphas (np.ndarray): Support vector Lagrange multipliers.
        sv (np.ndarray): Support vectors.
        sv_outputs (np.ndarray): Support vector outputs.
        b (float): Constant.
        inputs (np.ndarray): Inputs.
        outputs (np.ndarray): Outputs.
        Returns (float): Binary classification error percentage
        """
        x = inputs
        y = outputs
        num_sv = len(sv)
        
        gs = []
        for xm in x:
            signal = 0.0
            for n in range(0, num_sv):
                if self.rbf:
                    kernel = self.kernel_rbf(sv[n], xm)
                else:
                    kernel = self.kernel_poly(sv[n], xm)
                signal += sv_alphas[n] * sv_outputs[n] * kernel
            signal += b
            gs.append(signal)
                
        g = np.array(np.sign(gs))
        return 100. * np.sum(y != g) / len(y)     
    
    # privates
    def kernel_poly(self, xn, xm):
        return (1.0 + np.dot(xn.T, xm))**self.Q
    
    def kernel_rbf(self, xn, xm):
        gamma = 1.0
        xn_square = np.dot(xn.T, xn)
        xm_square = np.dot(xm.T, xm)
        #return np.exp(-gamma * (xn_square + xm_square - 2 * np.dot(xn.T, xm)))
        return np.exp(-gamma * (np.linalg.norm(xn - xm)**2))
    
    def transform_inputs(self, inputs):
        return np.insert(inputs, 0, 1.0, axis=1)<jupyter_output><empty_output><jupyter_text>Compute in-sample error and number of support vectors for digit classifiers with C = 0.01 and Q = 2<jupyter_code>def in_out_sample_errors(classifiers, features_train, features_test=None, Cs=[0.01], Qs=[2], kernel='poly', 
                          gamma=1.0, coef0=1.0):
    for class_label in classifiers.keys():
        for C in Cs:
            for Q in Qs:
                dataset_train = get_dataset(features_train, class_label)
                inputs_train = dataset_train[:, 0:2]
                outputs_train = dataset_train[:, 2]
                
                if kernel == 'poly':
                    clf = svm.SVC(C=C, kernel=kernel, gamma=gamma, coef0=coef0, degree=Q)
                elif kernel == 'rbf':
                    clf = svm.SVC(C=C, kernel=kernel, gamma=gamma)
                clf.fit(inputs_train, outputs_train)
                
                error_in_sample = 1.0 - clf.score(inputs_train, outputs_train)
                
                if not features_test.empty:
                    dataset_test = get_dataset(features_test, class_label)
                    inputs_test = dataset_test[:, 0:2]
                    outputs_test = dataset_test[:, 2]
                    
                    error_out_sample = 1.0 - accuracy_score(outputs_test, clf.predict(inputs_test))
                
                if features_test.empty:
                     print('For C={0} and the polynomial kernel (Q={1}), the "{2}" classifier in-sample error Ein = {3}%, '
                           'and number of support vectors = {4}.'
                           .format(C, Q, class_label, round(error_in_sample, 3), len(clf.support_vectors_)))
                elif kernel == 'poly':
                    print('For C={0} and the polynomial kernel (Q={1}), the "{2}" classifier in-sample error Ein = {3}%, ' 
                          'out-of-sample error Eout = {4}% and number of support vectors = {5}.'
                          .format(C, Q, class_label, round(error_in_sample, 3), round(error_out_sample, 3), 
                                  len(clf.support_vectors_)))
                elif kernel == 'rbf':
                    print('For C={0}, and the RBF kernel (γ={1}), the "{2}" classifier in-sample error Ein = {3}% and ' 
                          'out-of-sample error Eout = {4}%'
                          .format(C, gamma, class_label, round(error_in_sample, 3), round(error_out_sample, 3)))
in_out_sample_errors(classifiers_one_vs_all, features_train_one_vs_all, pd.DataFrame(), Cs=[0.01], Qs=[2], kernel='poly')<jupyter_output>For C=0.01 and the polynomial kernel (Q=2), the "zero_vs_all" classifier in-sample error Ein = 0.106%, and number of support vectors = 2179.
For C=0.01 and the polynomial kernel (Q=2), the "one_vs_all" classifier in-sample error Ein = 0.014%, and number of support vectors = 386.
For C=0.01 and the polynomial kernel (Q=2), the "two_vs_all" classifier in-sample error Ein = 0.1%, and number of support vectors = 1970.
For C=0.01 and the polynomial kernel (Q=2), the "three_vs_all" classifier in-sample error Ein = 0.09%, and number of support vectors = 1950.
For C=0.01 and the polynomial kernel (Q=2), the "four_vs_all" classifier in-sample error Ein = 0.089%, and number of support vectors = 1856.
For C=0.01 and the polynomial kernel (Q=2), the "five_vs_all" classifier in-sample error Ein = 0.076%, and number of support vectors = 1585.
For C=0.01 and the polynomial kernel (Q=2), the "six_vs_all" classifier in-sample error Ein = 0.091%, and number of support vectors = 1893.
For C=0.01 and the [...]<jupyter_text>Set output labels for one-vs-one classifiers<jupyter_code>classifiers_one_vs_one = dict({'one_vs_five': [1,5]})<jupyter_output><empty_output><jupyter_text>Create one-versus-one dataframe by adding outputs for the different classifiers to the training and testing dataframes<jupyter_code>def create_one_vs_one_dataframe(df, classifiers):
    # add binary labels to dataframe  
    for class_label in classifiers.keys():
        digits = classifiers[class_label]
        one_vs_one = pd.DataFrame(df.loc[df['digit'].isin(digits),:], copy=True)
        for digit in digits:
            labels = one_vs_one.loc[one_vs_one['digit'] == digit, 'digit']
            labels.loc[:] = 1.0
            one_vs_one[class_label] = labels
            break
        
    one_vs_one.fillna(-1.0, inplace=True)
    return one_vs_one
features_train_one_vs_one = create_one_vs_one_dataframe(features_train, classifiers_one_vs_one)
features_test_one_vs_one = create_one_vs_one_dataframe(features_test, classifiers_one_vs_one)<jupyter_output><empty_output><jupyter_text>Compute in-sample error, out-of-sample error and number of support vectors for 1 vs 5 classifier with C = {0.001, 0.01, 0.1, 1} and Q = {2, 5}<jupyter_code>in_out_sample_errors(classifiers_one_vs_one, features_train_one_vs_one, features_test_one_vs_one, 
                     Cs=[0.001, 0.01, 0.1, 1.0], Qs=[2, 5], kernel='poly')<jupyter_output>For C=0.001 and the polynomial kernel (Q=2), the "one_vs_five" classifier in-sample error Ein = 0.004%, out-of-sample error Eout = 0.017% and number of support vectors = 76.
For C=0.001 and the polynomial kernel (Q=5), the "one_vs_five" classifier in-sample error Ein = 0.004%, out-of-sample error Eout = 0.021% and number of support vectors = 25.
For C=0.01 and the polynomial kernel (Q=2), the "one_vs_five" classifier in-sample error Ein = 0.004%, out-of-sample error Eout = 0.019% and number of support vectors = 34.
For C=0.01 and the polynomial kernel (Q=5), the "one_vs_five" classifier in-sample error Ein = 0.004%, out-of-sample error Eout = 0.021% and number of support vectors = 23.
For C=0.1 and the polynomial kernel (Q=2), the "one_vs_five" classifier in-sample error Ein = 0.004%, out-of-sample error Eout = 0.019% and number of support vectors = 24.
For C=0.1 and the polynomial kernel (Q=5), the "one_vs_five" classifier in-sample error Ein = 0.003%, out-of-sample error Eout = 0.019[...]<jupyter_text>Cross Validation Compute 10-fold cross-validation error Ecv and the value of C with the lowest Ecv over 100 runs for 1 vs 5 classifier with C = {0.0001, 0.001, 0.01, 0.1, 1} and Q = 2<jupyter_code>def cross_val_error(classifiers, features_train, Cs, num_folds, shuffle, num_runs):
    errors_cv = OrderedDict.fromkeys(Cs, 0.0)
    low_cv_counts = OrderedDict.fromkeys(Cs, 0)
    
    for class_label in classifiers.keys():
        dataset_train = get_dataset(features_train, class_label)
        inputs_train = dataset_train[:, 0:2]
        outputs_train = dataset_train[:, 2]
        
        for i in range(0, num_runs):
            k_fold = KFold(n_splits=num_folds, shuffle=shuffle)
            run_errors_cv = []
            for C in Cs:
                clf = svm.SVC(C=C, kernel='poly', gamma=1.0, coef0=1.0, degree=2)
                scores = cross_val_score(clf, inputs_train, outputs_train, cv=k_fold)
                #print('scores = ', scores)
                error_cv = 1.0 - scores.mean()
                errors_cv[C] += error_cv
                run_errors_cv.append(error_cv)
            
            min_error_cv = np.argmin(run_errors_cv)
            low_cv_counts[list(low_cv_counts.keys())[min_error_cv]] += 1
            
            run_num = i+1
            #print('Run number {0} completed, C = {1} selected, error = {2}'
            #      .format(run_num, list(low_cv_counts.keys())[min_error_cv], run_errors_cv[min_error_cv]))

        low_cv_count = max(low_cv_counts, key=low_cv_counts.get)
            
        print('C={0} is selected the most often ({1} times out of {2} runs) and the average value of the cross-validation'
              ' error Ecv = {3}'.format(low_cv_count, low_cv_counts[low_cv_count], num_runs, 
                                        round(errors_cv[low_cv_count] / num_runs, 3)))
cross_val_error(classifiers_one_vs_one, features_train_one_vs_one, Cs=[0.0001, 0.001, 0.01, 0.1, 1.0], num_folds=10, 
                shuffle=True, num_runs=100)<jupyter_output>C=0.001 is selected the most often (28 times out of 100 runs) and the average value of the cross-validation error Ecv = 0.005<jupyter_text>RBF Kernel Compute in-sample error, out-of-sample error and number of support vectors for 1 vs 5 classifier with C = {0.01, 1, 100, 10^4, 10^6} for radial basis function (RBF kernel)<jupyter_code>in_out_sample_errors(classifiers_one_vs_one, features_train_one_vs_one, features_test_one_vs_one, 
                     Cs=[0.01, 1., 100., 1.*10**4, 1.*10**6], kernel='rbf', gamma=1.0)<jupyter_output>For C=0.01, and the RBF kernel (γ=1.0), the "one_vs_five" classifier in-sample error Ein = 0.004% and out-of-sample error Eout = 0.024%
For C=1.0, and the RBF kernel (γ=1.0), the "one_vs_five" classifier in-sample error Ein = 0.004% and out-of-sample error Eout = 0.021%
For C=100.0, and the RBF kernel (γ=1.0), the "one_vs_five" classifier in-sample error Ein = 0.003% and out-of-sample error Eout = 0.019%
For C=10000.0, and the RBF kernel (γ=1.0), the "one_vs_five" classifier in-sample error Ein = 0.003% and out-of-sample error Eout = 0.024%
For C=1000000.0, and the RBF kernel (γ=1.0), the "one_vs_five" classifier in-sample error Ein = 0.001% and out-of-sample error Eout = 0.024%
{'fraction_non_alphanumeric': 0.12291940742435509, 'fraction_numerical': 0.02808442486987359, 'mean_word_length': 2.3315548780487805, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '348828', 'license': 'MIT', 'n_tokens_mistral': 6111, 'n_tokens_neox': 5675, 'n_words': 1498, 'path': 'Solutions/hw8_soln_code.ipynb', 'repo_name': 'covuworie/learning-from-data', 'size': '17483'}