<jupyter_start><jupyter_text>Chapter 5 - Resampling Methods - [Load dataset](Load-dataset)- [Cross-Validation](5.1-Cross-Validation)<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import sklearn.linear_model as skl_lm
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split, LeaveOneOut, KFold, cross_val_score
from sklearn.preprocessing import PolynomialFeatures

%matplotlib inline
#plt.style.use('seaborn-white')
plt.style.use('ggplot')
import warnings
warnings.simplefilter('ignore')<jupyter_output><empty_output><jupyter_text>Load dataset Dataset available on http://www-bcf.usc.edu/~gareth/ISL/data.html<jupyter_code>df1 = pd.read_csv('https://raw.githubusercontent.com/borisgarbuzov/schulich_data_science_1/master/Data/Auto.csv', na_values='?').dropna()
df1.info()
df1<jupyter_output><empty_output><jupyter_text>5.1 Cross-Validation  Figure 5.2 - Validation Set ApproachUsing Polynomial feature generation in scikit-learnhttp://scikit-learn.org/dev/modules/preprocessing.htmlgenerating-polynomial-features Перенес lab в другой ноутбукhttps://colab.research.google.com/drive/1DLrcBSMiz7kuxA85cSA5Hy5qcB4h1f5ZscrollTo=PkBjEP_gF92R<jupyter_code>t_prop = 0.5
p_order = np.arange(1,11) # polynomial degree
r_state = np.arange(0,10) # every seed splplits differently
X, Y = np.meshgrid(p_order, r_state, indexing='ij') # constant rows and constant cols
X, Y
Z = np.zeros((p_order.size,r_state.size)) # allocate memory for MSE

regr = skl_lm.LinearRegression()

# Generate 10 random splits of the dataset
for (i,j),v in np.ndenumerate(Z):
    # 2 matrices - degrees and states
    # i, j go over both of them. 
    # in case 1 i is constant
    # in case 2 j is constant
    # Stopped here
    poly = PolynomialFeatures(int(X[i,j]))
    # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html
    X_poly = poly.fit_transform(df1.horsepower.values.reshape(-1,1))
    
    X_train, X_test, y_train, y_test = train_test_split(X_poly, df1.mpg.ravel(),
                                                        test_size=t_prop, random_state=Y[i,j])
                                                                        
    regr.fit(X_train, y_train)
    pred = regr.predict(X_test)
    Z[i,j]= mean_squared_error(y_test, pred)
    # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html

fig, (ax1, ax2) = plt.subplots(1,2, figsize=(10,4))

# Left plot (first split)
ax1.plot(X.T[0],Z.T[0], '-o')
ax1.set_title('Random split of the data set')

# Right plot (all splits)
ax2.plot(X,Z)
ax2.set_title('10 random splits of the data set')

for ax in fig.axes:
    ax.set_ylabel('Mean Squared Error')
    ax.set_ylim(15,30)
    ax.set_xlabel('Degree of Polynomial')
    ax.set_xlim(0.5,10.5)
    ax.set_xticks(range(2,11,2));<jupyter_output><empty_output><jupyter_text>Figure 5.4<jupyter_code>p_order = np.arange(1,11)
r_state = np.arange(0,10)

# LeaveOneOut CV
regr = skl_lm.LinearRegression()
loo = LeaveOneOut()
# https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.LeaveOneOut.html
loo.get_n_splits(df1)
scores = list()

for i in p_order:
    poly = PolynomialFeatures(i)
    X_poly = poly.fit_transform(df1.horsepower.values.reshape(-1,1))
    score = cross_val_score(regr, X_poly, df1.mpg, cv=loo, scoring='neg_mean_squared_error').mean()
    # can it work with some dafault scoring?
    # score = cross_val_score(regr, X_poly, df1.mpg, cv=loo).mean()
    # Did not work. Printed an array of nans and warnings about R^2
    scores.append(score)
scores
# k-fold CV
folds = 10
elements = len(df1.index)

X, Y = np.meshgrid(p_order, r_state, indexing='ij')
Z = np.zeros((p_order.size,r_state.size))

regr = skl_lm.LinearRegression()

for (i,j),v in np.ndenumerate(Z):
    poly = PolynomialFeatures(X[i,j])
    X_poly = poly.fit_transform(df1.horsepower.values.reshape(-1,1))
    kf_10 = KFold(n_splits=folds, random_state=None)
    # kf_10 = KFold(n_splits=folds, random_state=Y[i,j])
    Z[i,j] = cross_val_score(regr, X_poly, df1.mpg, cv=kf_10, scoring='neg_mean_squared_error').mean()    
fig, (ax1, ax2) = plt.subplots(1,2, figsize=(10,4))

# Note: cross_val_score() method return negative values for the scores.
# https://github.com/scikit-learn/scikit-learn/issues/2439

# Left plot
ax1.plot(p_order, np.array(scores)*-1, '-o')
ax1.set_title('LOOCV')

# Right plot
ax2.plot(X,Z*-1)
ax2.set_title('10-fold CV')

for ax in fig.axes:
    ax.set_ylabel('Mean Squared Error')
    ax.set_ylim(15,30)
    ax.set_xlabel('Degree of Polynomial')
    ax.set_xlim(0.5,10.5)
    ax.set_xticks(range(2,11,2));<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13192389006342495, 'fraction_numerical': 0.02684989429175476, 'mean_word_length': 4.370034052213394, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '532059', 'license': 'MIT', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1810, 'n_words': 332, 'path': 'Chapter 5/Python/cross-validation/Tibshirani ch5 resampling.ipynb', 'repo_name': 'borisgarbuzov/schulich_data_science_1', 'size': '4730'}