<jupyter_start><jupyter_text>Chapter 4 - Classification: Applied Exercises<jupyter_code># loading required packages

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sn
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB, BernoulliNB,  MultinomialNB # might need different naive Bayes classifier<jupyter_output><empty_output><jupyter_text>Exercise 13<jupyter_code># loading the data

weekly = pd.read_csv('data/Weekly.csv')
weekly.head()
# replace down with 0 / up with 1 

weekly.Direction.replace(('Up', 'Down'), (1,0), inplace=True)
weekly.head()
# a) numerical and graphical summaries of the Weekly data

corrMatrix = weekly.iloc[:,1:9].corr()
sn.heatmap(corrMatrix, annot=True)
plt.show()

weekly.iloc[:,1:9].describe()
variables = ['Lag1', 'Lag2', 'Lag3', 'Lag4', 'Lag5', 'Volume']

fig, axs = plt.subplots(nrows=2, ncols=3, figsize=(12,12))

for variable, ax in zip(variables, axs.flatten()):
    sn.boxplot(x='Direction', y=variable, data=weekly, ax=ax)


# b) logistic regression - 'Direction' as response variable

y = weekly['Direction']
X = weekly.iloc[:,1:7].to_numpy()

logReg = LogisticRegression(penalty='none').fit(X, y)

coef_names = ['Intercept']
[coef_names.append(i) for i in weekly.columns.tolist()[1:7]]

coef_values = [logReg.intercept_[0]]
[coef_values.append(i) for i in logReg.coef_[0]]

coefs = dict(zip(coef_names, coef_values))
 
for key, value in coefs.items():
    print(key, ': ', value)
    
# -> Coefficients match values in R; sklearn can't output p-values -> use statsmodels instead?
# logReg using statsmodels

import statsmodels.api as sm

X2 = sm.add_constant(X)

logReg2 = sm.Logit(y, X2).fit()
print(logReg2.summary())<jupyter_output>Optimization terminated successfully.
         Current function value: 0.682441
         Iterations 4
                           Logit Regression Results                           
==============================================================================
Dep. Variable:              Direction   No. Observations:                 1089
Model:                          Logit   Df Residuals:                     1082
Method:                           MLE   Df Model:                            6
Date:                Thu, 14 Oct 2021   Pseudo R-squ.:                0.006580
Time:                        14:49:26   Log-Likelihood:                -743.18
converged:                       True   LL-Null:                       -748.10
Covariance Type:            nonrobust   LLR p-value:                    0.1313
==============================================================================
                 coef    std err          z      P>|z|      [0.025      0.975]
-----------------------------[...]<jupyter_text>Only the intercept and Lag2 are statistically significant at $\alpha = 0.05$.<jupyter_code># c) confusion matrix / correct predictions

y_pred = logReg.predict(X)

def model_eval(y_test, y_pred):
    cm = confusion_matrix(y_test, y_pred)
    print(cm)


    correctly_classified = cm[0][0] + cm[1][1]
    total = np.sum(cm)
    percentage = round(correctly_classified/total*100, 4)

    print('Correctly classified values in percent:\t' + str(percentage) +'%')
    
model_eval(y, y_pred)
# d) logistic regression (train/test split)

trainData = weekly[(weekly['Year'] >= 1990) & (weekly['Year'] <= 2008)]
testData = weekly[(weekly['Year'] >= 2009) & (weekly['Year'] <= 2010)]

y_train = trainData['Direction']
y_test = testData['Direction']

X_train = trainData.iloc[:,1:7]
X_test = testData.iloc[:,1:7]

# fitting the model

logReg = LogisticRegression(penalty='none').fit(X_train, y_train)

# predicting test Values

y_pred = logReg.predict(X_test)
    
model_eval(y_test, y_pred)
# e) LDA

LDA = LinearDiscriminantAnalysis().fit(X_train, y_train)

y_predLDA = LDA.predict(X_test)

model_eval(y_test, y_predLDA)
# f) QDA

QDA = QuadraticDiscriminantAnalysis().fit(X_train, y_train)

y_predQDA = QDA.predict(X_test)

model_eval(y_test, y_predQDA)
# g) KNN

KNN = KNeighborsClassifier(n_neighbors=1).fit(X_train, y_train)

y_predKNN = KNN.predict(X_test)

model_eval(y_test, y_predKNN)
# h) naive Bayes

NB_B = BernoulliNB().fit(X_train, y_train)

y_pred_NB_B = NB_B.predict(X_test)

model_eval(y_test, y_pred_NB_B)<jupyter_output>[[12 31]
 [14 47]]
Correctly classified values in percent:	56.7308%<jupyter_text>i) Naive Bayes performs the best on this dataset.<jupyter_code># j) experiment: Different combinations of predictors / transformations / interactions for each of the methods

# train / test splot with CV
# KNN -> different number of neighbors
# penalties logistic regression<jupyter_output><empty_output><jupyter_text>Exercise 14<jupyter_code># loading the data

auto = pd.read_csv('data/Auto.csv')
auto.head()
# a) create binary variable 'mpg01'
auto['mpg01'] = np.where(auto['mpg'] >= auto['mpg'].median(), 1, 0)
auto.head()
# b) explore the data graphically

corrMatrix = auto.iloc[:, np.r_[1:7, 9]].corr()

sn.heatmap(corrMatrix, annot=True)
plt.show()
variables = auto.columns.tolist()[2:6]

fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(12,12))

for variable, ax in zip(variables, axs.flatten()):
    sn.boxplot(x='mpg01', y=variable, data=auto, ax=ax)
# c) Split the data into a train / test set

X = auto.iloc[:,1:5]
y = auto['mpg01']

X_train, X_test, y_train, y_test =  train_test_split(X, y, test_size=0.33, random_state=42)
# d) LDA -> test error?

LDA = LinearDiscriminantAnalysis().fit(X_train, y_train)

y_predLDA = LDA.predict(X_test)

def error_rate(y_test, y_pred):
    cm = confusion_matrix(y_test, y_pred)
    print(cm)


    correctly_classified = cm[0][0] + cm[1][1]
    total = np.sum(cm)
    percentage = correctly_classified/total*100
    error_rate = round(100 - percentage, 4)

    print('Error rate in percent:\t' + str(error_rate) +'%')
    
error_rate(y_test, y_predLDA)
# e) QDA -> test error?

QDA = QuadraticDiscriminantAnalysis().fit(X_train, y_train)

y_predQDA = QDA.predict(X_test)

error_rate(y_test, y_predQDA)
# f) logistic regression -> test error?

logReg = LogisticRegression(penalty='none').fit(X_train, y_train)

y_predLR = logReg.predict(X_test)

error_rate(y_test, y_predLR)
# g) naive Bayes -> test error?

NB_B = BernoulliNB().fit(X_train, y_train)

y_pred_NB_B = NB_B.predict(X_test)

error_rate(y_test, y_pred_NB_B)
# h) KNN -> test error? Which value of K performs the best on the data set?

for k in range(1,9):
    print(f'k = {k}')
    KNN = KNeighborsClassifier(n_neighbors=k).fit(X_train, y_train)

    y_predKNN = KNN.predict(X_test)

    error_rate(y_test, y_predKNN)<jupyter_output>k = 1
[[53 17]
 [ 3 57]]
Error rate in percent:	15.3846%
k = 2
[[57 13]
 [ 6 54]]
Error rate in percent:	14.6154%
k = 3
[[52 18]
 [ 1 59]]
Error rate in percent:	14.6154%
k = 4
[[53 17]
 [ 3 57]]
Error rate in percent:	15.3846%
k = 5
[[49 21]
 [ 1 59]]
Error rate in percent:	16.9231%
k = 6
[[50 20]
 [ 1 59]]
Error rate in percent:	16.1538%
k = 7
[[48 22]
 [ 1 59]]
Error rate in percent:	17.6923%
k = 8
[[48 22]
 [ 1 59]]
Error rate in percent:	17.6923%<jupyter_text>Use k = 2 or k = 3  Exercise 15<jupyter_code># a) Power() function -> 2^3 and print result

def Power():
    print(2**3)
    
Power()
# b) Power2() function -> x^a print result

def Power2(x, a):
    print(x**a)

Power2(3, 8)
# c) compute 10^3, 8^17, 131^3 using Power2()

Power2(10, 3)
Power2(8, 17)
Power2(131, 3)
# d) Power3() function -> x^a return result

def Power3(x, a):
    return x**a
# e) create plot for f(x)=x^2 with Power3(); x-axis 1-10, y-axis x^2

x = list(range(11))
y = [Power3(i, 2) for i in x]

plt.plot(x,y)
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.show()
# f) PlotPower() function -> create plot of x agains x^a for a fixed a and range of values of x

def PlotPower(x1, x2, a):
    x = []
    for i in range(x1, x2+1):
        x.append(i)
        
    y = [i**a for i in x]
    
    plt.plot(x,y)
    plt.xlabel('$x$')
    plt.ylabel('$f(x)$')
    plt.show()

# example
PlotPower(5, 12, 3)<jupyter_output><empty_output><jupyter_text>Exercise 16<jupyter_code># loading the data

boston = pd.read_csv('data/Boston.csv')
boston.head()
# binary variable

boston['crim01'] = np.where(boston['crim'] >= boston['crim'].median(), 1, 0)
boston.head()
corrMatrix = boston.iloc[:, 1:].corr()

fig, ax = plt.subplots(figsize=(10,10))   
sn.heatmap(corrMatrix, annot=True, ax=ax)
plt.show()
X = boston.iloc[:, np.r_[1,2,4,6:10,11]]
y = boston['crim01']

X_train, X_test, y_train, y_test =  train_test_split(X, y, test_size=0.33, random_state=42)
model_names = ['LR', 'LDA', 'NB']

classifiers = [LogisticRegression(penalty='none', max_iter=1000), 
              LinearDiscriminantAnalysis(), 
              BernoulliNB()]

# Add names and model object initializations for KNN.
for k in range(1,6):
    model_names.append('KNN: k=' + str(k))
    classifiers.append(KNeighborsClassifier(n_neighbors=k))

# Fit and calculate predictions for each classifier in the list.
for name, classifier in zip(model_names, classifiers):
    model = classifier.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    print(name)
    error_rate(y_test, y_pred)<jupyter_output>LR
[[72 12]
 [21 62]]
Error rate in percent:	19.7605%
LDA
[[81  3]
 [28 55]]
Error rate in percent:	18.5629%
NB
[[36 48]
 [ 3 80]]
Error rate in percent:	30.5389%
KNN: k=1
[[78  6]
 [ 9 74]]
Error rate in percent:	8.982%
KNN: k=2
[[83  1]
 [15 68]]
Error rate in percent:	9.5808%
KNN: k=3
[[81  3]
 [12 71]]
Error rate in percent:	8.982%
KNN: k=4
[[81  3]
 [13 70]]
Error rate in percent:	9.5808%
KNN: k=5
[[79  5]
 [13 70]]
Error rate in percent:	10.7784%
{'fraction_non_alphanumeric': 0.15606351775058158, 'fraction_numerical': 0.04915545666026095, 'mean_word_length': 3.8257686676427527, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '299648', 'license': 'MIT', 'n_tokens_mistral': 4111, 'n_tokens_neox': 3706, 'n_words': 934, 'path': 'Chapter 4/Chapter4.ipynb', 'repo_name': 'SvenNekula/StatisticalLearning', 'size': '9887'}