<jupyter_start><jupyter_text>Pondering the Problem of Fitting a Model  Dividing between training, test and validation<jupyter_code>from sklearn.datasets import load_boston
boston = load_boston()
X, y = boston.data, boston.target
print(X.shape, y.shape)
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
regression = LinearRegression()
regression.fit(X,y)
print('Mean squared error: %.2f' % mean_squared_error(
    y_true=y, y_pred=regression.predict(X)))
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.30, random_state=5)
print(X_train.shape, X_test.shape)
regression.fit(X_train,y_train)
print('Train mean squared error: %.2f' % mean_squared_error(
    y_true=y_train, y_pred=regression.predict(X_train)))
print('Test mean squared error: %.2f' % mean_squared_error(
    y_true=y_test, y_pred=regression.predict(X_test)))
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.30, random_state=6)
regression.fit(X_train,y_train)
print('Train mean squared error: %.2f' % mean_squared_error(
    y_true=y_train, y_pred=regression.predict(X_train)))
print('Test mean squared error: %.2f' % mean_squared_error(
    y_true=y_test, y_pred=regression.predict(X_test)))<jupyter_output>Train mean squared error: 19.48
Test mean squared error: 28.33<jupyter_text>Cross-validating  Using cross-validation on k folds<jupyter_code>from sklearn.model_selection import cross_val_score, KFold
import numpy as np
crossvalidation = KFold(n_splits=10, shuffle=True, random_state=1)
scores = cross_val_score(regression, X, y, 
    scoring='neg_mean_squared_error', cv=crossvalidation, n_jobs=1)
print('Folds: %i, mean squared error: %.2f std: %.2f' % 
      (len(scores),np.mean(np.abs(scores)),np.std(scores)))<jupyter_output>Folds: 10, mean squared error: 23.76 std: 12.13<jupyter_text>Sampling stratifications for complex data<jupyter_code>%matplotlib inline
import pandas as pd
df = pd.DataFrame(X, columns=boston.feature_names)
df['target'] = y
df.boxplot('target', by='CHAS', return_type='axes');
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import mean_squared_error
strata = StratifiedShuffleSplit(n_splits=3, 
                                test_size=0.35, 
                                random_state=0)
scores = list()
for train_index, test_index in strata.split(X, X[:,3]):
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]
    regression.fit(X_train, y_train)
    scores.append(mean_squared_error(y_true=y_test, 
                       y_pred=regression.predict(X_test)))
print('%i folds cv mean squared error: %.2f std: %.2f' % 
      (len(scores),np.mean(np.abs(scores)),np.std(scores)))<jupyter_output>3 folds cv mean squared error: 24.30 std: 3.99<jupyter_text>Selecting Variables Like a Pro  Selecting on univariate measures<jupyter_code>from sklearn.feature_selection import SelectPercentile
from sklearn.feature_selection import f_regression
Selector_f = SelectPercentile(f_regression, percentile=25)
Selector_f.fit(X, y)
for n,s in zip(boston.feature_names,Selector_f.scores_):
    print('F-score: %3.2f\t for feature %s ' % (s,n))<jupyter_output>F-score: 88.15	 for feature CRIM 
F-score: 75.26	 for feature ZN 
F-score: 153.95	 for feature INDUS 
F-score: 15.97	 for feature CHAS 
F-score: 112.59	 for feature NOX 
F-score: 471.85	 for feature RM 
F-score: 83.48	 for feature AGE 
F-score: 33.58	 for feature DIS 
F-score: 85.91	 for feature RAD 
F-score: 141.76	 for feature TAX 
F-score: 175.11	 for feature PTRATIO 
F-score: 63.05	 for feature B 
F-score: 601.62	 for feature LSTAT<jupyter_text>Using a greedy search<jupyter_code>from sklearn.feature_selection import RFECV
selector = RFECV(estimator=regression, 
                 cv=10, 
                 scoring='neg_mean_squared_error')
selector.fit(X, y)
print("Optimal number of features : %d" 
      % selector.n_features_)
print(boston.feature_names[selector.support_])<jupyter_output>['CHAS' 'NOX' 'RM' 'DIS' 'PTRATIO' 'LSTAT']<jupyter_text>Pumping up your hyper-parameters<jupyter_code>import numpy as np
from sklearn.datasets import load_iris
iris = load_iris()
X, y = iris.data, iris.target<jupyter_output><empty_output><jupyter_text>Implementing a grid search<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
classifier = KNeighborsClassifier(n_neighbors=5, 
      weights='uniform', metric= 'minkowski', p=2)
grid = {'n_neighbors': range(1,11), 
        'weights': ['uniform', 'distance'], 'p': [1,2]}
print ('Number of tested models: %i' 
       % np.prod([len(grid[element]) for element in grid]))
score_metric = 'accuracy'
from sklearn.model_selection import cross_val_score
print('Baseline with default parameters: %.3f' 
      % np.mean(cross_val_score(classifier, X, y, 
                cv=10, scoring=score_metric, n_jobs=1)))
from sklearn.model_selection import GridSearchCV
search = GridSearchCV(estimator=classifier, 
                      param_grid=grid, 
                      scoring=score_metric, 
                      n_jobs=1, 
                      refit=True, 
                      return_train_score=True, 
                      cv=10)
search.fit(X,y)
print('Best parameters: %s' % search.best_params_)
print('CV Accuracy of best parameters: %.3f' % 
      search.best_score_)
print(search.cv_results_)
from sklearn.model_selection import validation_curve
model = KNeighborsClassifier(weights='uniform', 
                             metric= 'minkowski', p=1)
train, test = validation_curve(model, X, y, 
                               param_name='n_neighbors', 
                               param_range=range(1, 11), 
                               cv=10, scoring='accuracy', 
                               n_jobs=1)
import matplotlib.pyplot as plt
mean_train  = np.mean(train,axis=1)
mean_test   = np.mean(test,axis=1)
plt.plot(range(1,11), mean_train,'ro--', label='Training')
plt.plot(range(1,11), mean_test,'bD-.', label='CV')
plt.grid()
plt.xlabel('Number of neighbors')
plt.ylabel('accuracy')
plt.legend(loc='upper right', numpoints= 1)
plt.show()<jupyter_output><empty_output><jupyter_text>Trying a randomized search<jupyter_code>from sklearn.model_selection import RandomizedSearchCV
random_search = RandomizedSearchCV(estimator=classifier, 
                    param_distributions=grid, n_iter=10, 
    scoring=score_metric, n_jobs=1, refit=True, cv=10, )
random_search.fit(X, y)
print('Best parameters: %s' % random_search.best_params_)
print('CV Accuracy of best parameters: %.3f' % 
      random_search.best_score_)<jupyter_output>Best parameters: {'weights': 'distance', 'p': 2, 'n_neighbors': 5}
CV Accuracy of best parameters: 0.967
{'fraction_non_alphanumeric': 0.11417725215359907, 'fraction_numerical': 0.023361074609432035, 'mean_word_length': 4.269230769230769, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '394388', 'license': 'MIT', 'n_tokens_mistral': 2546, 'n_tokens_neox': 2330, 'n_words': 511, 'path': 'PDS Resources/P4DS4D2_18_Performing_Cross_Validation_Selection_Optimization.ipynb', 'repo_name': 'jay6430/Python-for-Data-Science', 'size': '6849'}