<jupyter_start><jupyter_text>Helper tools<jupyter_code>import scipy.io as sio
from scipy.stats import multivariate_normal
import numpy as np
import itertools
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix
%matplotlib inline
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

#     print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')<jupyter_output><empty_output><jupyter_text>Exercise 3<jupyter_code>train = sio.loadmat('breast_cancer/breast_cancer_train.mat')['breast_cancer_train']
val = sio.loadmat('breast_cancer/breast_cancer_val.mat')['breast_cancer_val']
labels = {'b': 2, 'm': 4}
names, counts = np.unique(train[:, 10], return_counts = True)
priors = counts / float(sum(counts))
print "priors =", zip(names, priors)
def estimate_pdf_class(features, class_label, pca = False, eigenvector = None, verbose = False):
    if pca:
        data = train[train[:, 10] == labels[class_label]][:, 1:10].dot(eigenvector)
    else:
        data = train[train[:, 10] == labels[class_label]][:, features]
    mean = np.mean(data, axis = 0)
    cov = np.cov(data.T)
    if verbose:
        print "columns = {}, \n class = {}".format(features, class_label)
        print "mean = {}, \n cov = \n {}".format(mean, cov)
        print('=====================')
    return multivariate_normal(mean = mean, cov = cov)

def posterior(x, pdf_b, pdf_m, priors, pca=False, eigenvector = None):
    if pca:
        x = x.dot(eigenvector)
    p_x_b = pdf_b.pdf(x)
    p_x_m = pdf_m.pdf(x)
    return p_x_b * priors[0] / (p_x_b * priors[0]  + p_x_m * priors[1])

def classify(point, posterior, pdf_b, pdf_m, priors, pca=False, eigenvector = None):
    posterior_c_b = posterior(point, pdf_b, pdf_m, priors, pca, eigenvector)
    return labels['b'] if posterior_c_b > 0.5 else labels['m']
def estimate_metrics(y_test, output):
    acc = accuracy_score(y_test, output)
    print "error = ", 1 - acc

    prec_m = precision_score(y_test, output, pos_label = labels['m'])
    rec_m = recall_score(y_test, output, pos_label = labels['m'])
    print "precision = {}, recall = {}".format(prec_m, rec_m)

    cm = confusion_matrix(y_test, output)
    plot_confusion_matrix(cm, ['b', 'm'])

def report_statistics(x_test, y_test, pdf_b, pdf_m, priors, pca=False, eigenvector = None):
    output = [classify(x, posterior, pdf_b, pdf_m, priors, pca, eigenvector) for x in x_test]
    estimate_metrics(y_test, output)<jupyter_output><empty_output><jupyter_text>Part a x2 (Clump Thickness) only<jupyter_code>features = 1
pdf_b = estimate_pdf_class(features, 'b', verbose = True)
pdf_m = estimate_pdf_class(features, 'm', verbose = True)<jupyter_output>columns = 1, 
 class = b
mean = 2.93609022556, 
 cov = 
 2.73929635409
=====================
columns = 1, 
 class = m
mean = 7.13286713287, 
 cov = 
 6.21461636955
=====================<jupyter_text>Statistics for train set (x2 clump Thickness only)<jupyter_code>report_statistics(train[:, features], train[:, 10], pdf_b, pdf_m, priors)<jupyter_output>error =  0.136919315403
precision = 0.906542056075, recall = 0.678321678322
Confusion matrix, without normalization<jupyter_text>Statistics for val set (x2 clump Thickness only)<jupyter_code>report_statistics(val[:, features], val[:, 10], pdf_b, pdf_m, priors)<jupyter_output>error =  0.14598540146
precision = 0.868421052632, recall = 0.6875
Confusion matrix, without normalization<jupyter_text>First principal component<jupyter_code>cov_total = np.cov(train[:, 1:10].T)
U, s, V = np.linalg.svd(cov_total)
print "eigenvalues = ", s
var_ratio = s[0] / sum(s)
print "variance covered = ", var_ratio
pdf_b = estimate_pdf_class("1 princ component", 'b', pca =  True, eigenvector = U[:, 0], verbose = True)
pdf_m = estimate_pdf_class("1 princ component", 'm', pca =  True, eigenvector = U[:, 0], verbose = True)
U[:, 2]<jupyter_output><empty_output><jupyter_text>Statistics for train set (first principal component)<jupyter_code>report_statistics(train[:, 1:10], train[:, 10], pdf_b, pdf_m, priors, pca=True, eigenvector = U[:, 0])<jupyter_output>error =  0.0366748166259
precision = 0.932432432432, recall = 0.965034965035
Confusion matrix, without normalization<jupyter_text>Statistics for val set (first principal component)<jupyter_code>report_statistics(val[:, 1:10], val[:, 10], pdf_b, pdf_m, priors, pca=True, eigenvector = U[:, 0])<jupyter_output>error =  0.014598540146
precision = 0.969387755102, recall = 0.989583333333
Confusion matrix, without normalization<jupyter_text>x2 and x7 (Bare Nuclei) only<jupyter_code>features = [1, 6]
pdf_b = estimate_pdf_class(features, 'b', verbose = True)
pdf_m = estimate_pdf_class(features, 'm', verbose = True)<jupyter_output>columns = [1, 6], 
 class = b
mean = [ 2.93609023  1.36466165], 
 cov = 
 [[ 2.73929635  0.2196198 ]
 [ 0.2196198   1.47029366]]
=====================
columns = [1, 6], 
 class = m
mean = [ 7.13286713  7.70629371], 
 cov = 
 [[ 6.21461637  0.08155225]
 [ 0.08155225  9.08214321]]
=====================<jupyter_text>Statistics for train set (x2 and x7 (Bare Nuclei) only)<jupyter_code>report_statistics(train[:, features], train[:, 10], pdf_b, pdf_m, priors)<jupyter_output>error =  0.0586797066015
precision = 0.916083916084, recall = 0.916083916084
Confusion matrix, without normalization<jupyter_text>Statistics for val set (x2 and x7 (Bare Nuclei) only)<jupyter_code>report_statistics(val[:, features], val[:, 10], pdf_b, pdf_m, priors)<jupyter_output>error =  0.0547445255474
precision = 0.917525773196, recall = 0.927083333333
Confusion matrix, without normalization<jupyter_text>First two principal components<jupyter_code>var_ratio = (s[0] + s[1]) / sum(s)
print "variance covered = ", var_ratio
pdf_b = estimate_pdf_class("1, 2 princ component", 'b', pca =  True, eigenvector = U[:, [0, 1]], verbose = True)
pdf_m = estimate_pdf_class("1, 2 princ component", 'm', pca =  True, eigenvector = U[:, [0, 1]], verbose = True)<jupyter_output>columns = 1, 2 princ component, 
 class = b
mean = [-4.71846346  0.65931253], 
 cov = 
 [[ 3.56621022 -0.12858286]
 [-0.12858286  0.93153259]]
=====================
columns = 1, 2 princ component, 
 class = m
mean = [-17.72960484   0.20953023], 
 cov = 
 [[ 19.70997548  -3.59290625]
 [ -3.59290625  12.9226298 ]]
=====================<jupyter_text>Statistics for train set (first two principal components)<jupyter_code>report_statistics(train[:, 1:10], train[:, 10], pdf_b, pdf_m, priors, pca=True, eigenvector = U[:, [0, 1]])<jupyter_output>error =  0.0366748166259
precision = 0.926666666667, recall = 0.972027972028
Confusion matrix, without normalization<jupyter_text>Statistics for val set  (first two principal components)<jupyter_code>report_statistics(val[:, 1:10], val[:, 10], pdf_b, pdf_m, priors, pca=True, eigenvector = U[:, [0, 1]])<jupyter_output>error =  0.021897810219
precision = 0.959183673469, recall = 0.979166666667
Confusion matrix, without normalization<jupyter_text>The principal components which explain 80% of the variance<jupyter_code>var_ratio = (s[0] + s[1] + s[2]) / sum(s)
print "variance covered = ", var_ratio
pdf_b = estimate_pdf_class("1, 2, 3 princ component", 'b', pca =  True, eigenvector = U[:, [0, 1, 2]], verbose = True)
pdf_m = estimate_pdf_class("1, 2, 3 princ component", 'm', pca =  True, eigenvector = U[:, [0, 1, 2]], verbose = True)<jupyter_output>columns = 1, 2, 3 princ component, 
 class = b
mean = [-4.71846346  0.65931253  1.05929594], 
 cov = 
 [[ 3.56621022 -0.12858286 -0.6351888 ]
 [-0.12858286  0.93153259  0.31212293]
 [-0.6351888   0.31212293  1.69077815]]
=====================
columns = 1, 2, 3 princ component, 
 class = m
mean = [-17.72960484   0.20953023   1.43781209], 
 cov = 
 [[ 19.70997548  -3.59290625   4.41095285]
 [ -3.59290625  12.9226298   -0.47097832]
 [  4.41095285  -0.47097832   9.05294776]]
=====================<jupyter_text>Statistics for train set (80% of the variance)<jupyter_code>report_statistics(train[:, 1:10], train[:, 10], pdf_b, pdf_m, priors, pca=True, eigenvector = U[:, [0, 1, 2]])<jupyter_output>error =  0.0366748166259
precision = 0.921052631579, recall = 0.979020979021
Confusion matrix, without normalization<jupyter_text>Statistics for val set (80% of the variance)<jupyter_code>report_statistics(val[:, 1:10], val[:, 10], pdf_b, pdf_m, priors, pca=True, eigenvector = U[:, [0, 1, 2]])<jupyter_output>error =  0.021897810219
precision = 0.959183673469, recall = 0.979166666667
Confusion matrix, without normalization<jupyter_text>all inputs (except sample code number)<jupyter_code>features = np.arange(1, 10)
pdf_b = estimate_pdf_class(features, 'b', verbose = True)
pdf_m = estimate_pdf_class(features, 'm', verbose = True)<jupyter_output>columns = [1 2 3 4 5 6 7 8 9], 
 class = b
mean = [ 2.93609023  1.30827068  1.42481203  1.35714286  2.07142857  1.36466165
  2.07518797  1.30827068  1.07518797], 
 cov = 
 [[  2.73929635e+00   3.97134345e-01   4.87629451e-01   3.36118598e-01
    1.93261456e-01   2.19619804e-01   2.99163002e-01   5.10341892e-01
    1.04979430e-03]
 [  3.97134345e-01   6.97063413e-01   6.00624202e-01   1.80053908e-01
    2.60916442e-01   4.26783941e-01   2.89941836e-01   5.38572847e-01
    3.33380621e-02]
 [  4.87629451e-01   6.00624202e-01   9.99985814e-01   1.94878706e-01
    2.78975741e-01   4.02993332e-01   2.20768903e-01   4.94963825e-01
    9.44814867e-03]
 [  3.36118598e-01   1.80053908e-01   1.94878706e-01   8.71967655e-01
    1.85714286e-01   4.39083558e-01   1.39083558e-01   2.93261456e-01
    5.98382749e-02]
 [  1.93261456e-01   2.60916442e-01   2.78975741e-01   1.85714286e-01
    7.45822102e-01   3.02156334e-01   1.07816712e-01   3.40161725e-01
   -5.39083558e-03]
 [  2.19619804e-01   4.26783[...]<jupyter_text>Statistics for train set (all inputs)<jupyter_code>report_statistics(train[:, features], train[:, 10], pdf_b, pdf_m, priors)<jupyter_output>error =  0.0488997555012
precision = 0.896774193548, recall = 0.972027972028
Confusion matrix, without normalization<jupyter_text>Statistics for val set  (all inputs)<jupyter_code>report_statistics(val[:, features], val[:, 10], pdf_b, pdf_m, priors)<jupyter_output>error =  0.0328467153285
precision = 0.930693069307, recall = 0.979166666667
Confusion matrix, without normalization<jupyter_text>Exercise 4<jupyter_code>from sklearn.linear_model import LogisticRegression
clf = LogisticRegression(C = 1)
clf.fit(train[:, 1:10], train[:, 10])
output_train = clf.predict(train[:, 1:10])
output_val = clf.predict(val[:, 1:10])<jupyter_output><empty_output><jupyter_text>Statistics for train set (logistic classifier)<jupyter_code>estimate_metrics(train[:, 10], output_train)<jupyter_output>error =  0.0342298288509
precision = 0.957446808511, recall = 0.944055944056
Confusion matrix, without normalization<jupyter_text>Statistics for val set (logistic classifier)<jupyter_code>estimate_metrics(val[:, 10], output_val)<jupyter_output>error =  0.021897810219
precision = 0.978723404255, recall = 0.958333333333
Confusion matrix, without normalization
{'fraction_non_alphanumeric': 0.14992105044461065, 'fraction_numerical': 0.14917310728829053, 'mean_word_length': 4.722301474084641, 'pattern_counts': {'":': 0, '<': 74, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '596606', 'license': 'MIT', 'n_tokens_mistral': 5678, 'n_tokens_neox': 4580, 'n_words': 987, 'path': 'hw/hw-3/hw_3.ipynb', 'repo_name': 'melaanya/twente-aml', 'size': '12033'}