<jupyter_start><jupyter_text>Model Evaluation & Fine-Tuning* Choosing the right model, and improving its performance<jupyter_code>import numpy as np
import pandas as pd
from sklearn import datasets
from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.utils import shuffle
from sklearn import metrics
from pandas.tools.plotting import scatter_matrix
from IPython.display import display

%matplotlib inline
boston = datasets.load_boston()
print(type(boston)) # A 'Bunch' is a special obj in python, similar to a dict
print(boston.keys())
print(boston.DESCR)
features = boston['feature_names']
boston_df = pd.DataFrame(boston['data'], columns=features)
boston_df['MEDV'] = pd.Series(boston['target'])
print(features)
target = pd.Series(boston['target'])
print('min, max = ', target.min(), ',', target.max())
print('mean = ', target.mean())
print('std = ', target.std())

# target values are in thousands of USD
# too many features to neatly visualize in a scatter mx. Let's pick a few we think will be good predictors
features_we_care_abt = ['RM', 'NOX', 'DIS', 'TAX', 'MEDV']
scatter_matrix(boston_df[features_we_care_abt], figsize=(12,8))
corr_mx = boston_df.corr()
corr_mx['MEDV'].sort_values(ascending=False)
data, target = shuffle(boston['data'], boston['target'])

X_train, X_test = data[:400], data[400:]
y_train, y_test = target[:400], target[400:]
print(X_train.shape, X_test.shape)
print(y_train.shape, y_test.shape)
lin_reg = LinearRegression()
d_tree = DecisionTreeRegressor()

lin_reg.fit(X_train, y_train)
d_tree.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Evaluate on Test Set<jupyter_code>y_pred_lin = lin_reg.predict(X_test)
y_pred_tree = d_tree.predict(X_test)

def RMSE(MSE):
    """Root-Mean-Squared Error"""
    return np.sqrt(abs(MSE))

MSE_lin = metrics.mean_squared_error(y_test, y_pred_lin)
MSE_tree = metrics.mean_squared_error(y_test, y_pred_tree)

print('LinReg MSE: ', MSE_lin)
print('D Tree MSE: ', MSE_tree)
print()
print('LinReg RMSE: ', RMSE(MSE_lin))
print('D Tree RMSE: ', RMSE(MSE_tree))<jupyter_output>LinReg MSE:  23.921011359005863
D Tree MSE:  19.000188679245284

LinReg RMSE:  4.8909110970253655
D Tree RMSE:  4.3589205864807035<jupyter_text>Cross Validation<jupyter_code>from sklearn.model_selection import cross_val_score

def display_scores(scores):
    print("Scores: ", scores)
    print("Mean: ", scores.mean())
    print("Std Dev: ", scores.std())
    print("Mean RMSE: ", RMSE(scores.mean()))

scores_tree = cross_val_score(d_tree, X_train, y_train, cv=5, scoring='neg_mean_squared_error')
scores_lin = cross_val_score(lin_reg, X_train, y_train, cv=5, scoring='neg_mean_squared_error')
print("Decision Tree:")
display_scores(scores_tree)
print("\nLinear Regression:")
display_scores(scores_lin)<jupyter_output>Decision Tree:
Scores:  [-34.65775  -18.84625  -22.027875 -12.64975  -16.240625]
Mean:  -20.884449999999998
Std Dev:  7.543750626594837
Mean RMSE:  4.569950765599121

Linear Regression:
Scores:  [-24.14632496 -32.64109898 -25.88726248 -20.64000621 -19.49767706]
Mean:  -24.562473936569155
Std Dev:  4.652533667922115
Mean RMSE:  4.956054270946713<jupyter_text>Fine Tuning with Grid Search<jupyter_code>from sklearn.model_selection import GridSearchCV

param_grid = [
    {'max_depth': [2,5,None], 'min_samples_leaf': [1, 5, 10, 20]}
]

grid_search = GridSearchCV(d_tree, param_grid, cv=5, scoring='neg_mean_squared_error')
grid_search.fit(X_train, y_train)
grid_search.best_params_
best_tree = grid_search.best_estimator_
best_tree.fit(X_train, y_train) # refit on whole training set
y_pred_gridSearch = best_tree.predict(X_test)

cv_res = pd.DataFrame(grid_search.cv_results_)
display(cv_res[['params', 'mean_train_score', 'std_train_score', 'mean_test_score', 'std_test_score']])

print('Grid Search MSE: ', metrics.mean_squared_error(y_test, y_pred_gridSearch))<jupyter_output>C:\ProgramData\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\util[...]<jupyter_text>Ensemble Methods* Averaging predictions from several models to get a better result* "Wisdom of the crowd"**Random Forest**: A collection of Decision Trees, all with randomized hyper-parameters<jupyter_code>from sklearn.ensemble import RandomForestRegressor

forest = RandomForestRegressor()
forest.fit(X_train, y_train)

scores_forest = cross_val_score(forest, X_train, y_train, cv=5, scoring='neg_mean_squared_error')
display_scores(scores_forest)

y_pred_forest = forest.predict(X_test)
MSE_forest = metrics.mean_squared_error(y_test, y_pred_forest)
print('Test MSE: ', MSE_forest)
print('Test RMSE: ', RMSE(MSE_forest))<jupyter_output>Scores:  [-16.5563875  -19.26936125 -14.41637625  -9.92418125  -8.88522125]
Mean:  -13.810305500000002
Std Dev:  3.926015648017051
Mean RMSE:  3.716221939012793
Test MSE:  15.458379245283018
Test RMSE:  3.931714542700553<jupyter_text>Grid Search CV on Random Forest<jupyter_code>param_grid_forest = [
    {'n_estimators': [3, 10, 30], 'max_features': [2,4,6,8]}
]

grid_search_forest = GridSearchCV(forest, param_grid_forest, cv=5, scoring='neg_mean_squared_error')
grid_search_forest.fit(X_train, y_train)
print(grid_search_forest.best_params_)
print(grid_search_forest.best_estimator_)
cv_res_forest = pd.DataFrame(grid_search_forest.cv_results_)
display(cv_res_forest[['params', 'mean_train_score', 'std_train_score', 'mean_test_score', 'std_test_score']])<jupyter_output>C:\ProgramData\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True
  warnings.warn(*warn_args, **warn_kwargs)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\util[...]<jupyter_text>Feature Importance<jupyter_code>feature_importances = grid_search_forest.best_estimator_.feature_importances_
feature_importances = dict(sorted(zip(feature_importances, features), reverse=True))
feature_importances
# let's try re-training our model on only the 'most important' features
data_pruned = boston_df[['LSTAT', 'RM', 'INDUS', 'PTRATIO']]
data_pruned = data_pruned.values
X_train_pruned, X_test_pruned = data_pruned[:400], data_pruned[400:]
print(X_train_pruned.shape, X_test_pruned.shape)
forest_p = RandomForestRegressor(max_features=6, n_estimators=30)
forest = RandomForestRegressor(max_features=6, n_estimators=30)

scores_forest = cross_val_score(forest, X_train, y_train, cv=5, scoring='neg_mean_squared_error')
scores_forest_p = cross_val_score(forest_p, X_train, y_train, cv=5, scoring='neg_mean_squared_error')

print('Pruned dataset: ')
display_scores(scores_forest_p)
print()
print('Unmodified dataset: ')
display_scores(scores_forest)<jupyter_output>Pruned dataset: 
Scores:  [-11.6461     -19.45809306 -12.33604611  -9.54396708  -8.12950056]
Mean:  -12.222741361111114
Std Dev:  3.9147444180065114
Mean RMSE:  3.4961037400384893

Unmodified dataset: 
Scores:  [-11.50920097 -19.05013944 -13.64189014  -8.44503444  -8.92704194]
Mean:  -12.314661388888885
Std Dev:  3.852971462756046
Mean RMSE:  3.5092251835538972<jupyter_text>Final performance evaluation on Test Set<jupyter_code>forest_p.fit(X_train, y_train)
y_pred = forest_p.predict(X_test)

MSE = metrics.mean_squared_error(y_test, y_pred)
print('final MSE: ', MSE)
print('final RMSE: ', RMSE(MSE))<jupyter_output>final MSE:  13.584440356394136
final RMSE:  3.6857075787959817
{'fraction_non_alphanumeric': 0.11561013386436553, 'fraction_numerical': 0.07854851200354021, 'mean_word_length': 7.15884476534296, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '526612', 'license': 'MIT', 'n_tokens_mistral': 3793, 'n_tokens_neox': 3242, 'n_words': 688, 'path': '04 Model Evaluation.ipynb', 'repo_name': 'charliedavenport/ACM_ML', 'size': '9039'}