<jupyter_start><jupyter_text>Hyper Parameter TuningOne of the primary objective and challenge in machine learning process is improving the performance score, based on data patterns and observed evidence. To achieve this objective, almost all machine learning algorithms have specific set of parameters that needs to estimate from dataset which will maximize the performance score. The best way to choose good hyperparameters is through trial and error of all possible combination of parameter values. Scikit-learn provide GridSearch and RandomSearch functions to facilitate automatic and reproducible approach for hyperparameter tuning.<jupyter_code>from IPython.display import Image
Image(filename='../Chapter 4 Figures/Hyper_Parameter_Tuning.png', width=1000)<jupyter_output><empty_output><jupyter_text>GridSearch<jupyter_code>import pandas as pd
import numpy as np

from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn import cross_validation
from sklearn import metrics

from matplotlib.colors import ListedColormap
import matplotlib.pyplot as plt
%matplotlib inline

from sklearn.ensemble import RandomForestClassifier
from sklearn.grid_search import GridSearchCV
seed = 2017

# read the data in
df = pd.read_csv("Data/Diabetes.csv")

X = df.ix[:,:8].values     # independent variables
y = df['class'].values     # dependent variables

#Normalize
X = StandardScaler().fit_transform(X)

# evaluate the model by splitting into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=seed)

kfold = cross_validation.StratifiedKFold(y=y_train, n_folds=5, random_state=seed)
num_trees = 100

clf_rf = RandomForestClassifier(random_state=seed).fit(X_train, y_train)

rf_params = {
    'n_estimators': [100, 250, 500, 750, 1000],
    'criterion':  ['gini', 'entropy'],
    'max_features': [None, 'auto', 'sqrt', 'log2'],
    'max_depth': [1, 3, 5, 7, 9]
}

# setting verbose = 10 will print the progress for every 10 task completion
grid = GridSearchCV(clf_rf, rf_params, scoring='roc_auc', cv=kfold, verbose=10, n_jobs=-1)
grid.fit(X_train, y_train)

print 'Best Parameters: ', grid.best_params_

results = cross_validation.cross_val_score(grid.best_estimator_, X_train,y_train, cv=kfold)
print "Accuracy - Train CV: ", results.mean()
print "Accuracy - Train : ", metrics.accuracy_score(grid.best_estimator_.predict(X_train), y_train)
print "Accuracy - Test : ", metrics.accuracy_score(grid.best_estimator_.predict(X_test), y_test)<jupyter_output>C:\Users\Manoh\Anaconda2\lib\site-packages\sklearn\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
C:\Users\Manoh\Anaconda2\lib\site-packages\sklearn\grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.
  DeprecationWarning)<jupyter_text>RandomSearch<jupyter_code>from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint as sp_randint

# specify parameters and distributions to sample from
param_dist = {'n_estimators':sp_randint(100,1000),
              'criterion': ['gini', 'entropy'],
              'max_features': [None, 'auto', 'sqrt', 'log2'],
              'max_depth': [None, 1, 3, 5, 7, 9]
             }

# run randomized search
n_iter_search = 20
random_search = RandomizedSearchCV(clf_rf, param_distributions=param_dist, cv=kfold, 
                                   n_iter=n_iter_search, verbose=10, n_jobs=-1, random_state=seed)

random_search.fit(X_train, y_train)
# report(random_search.cv_results_)

print 'Best Parameters: ', random_search.best_params_

results = cross_validation.cross_val_score(random_search.best_estimator_, X_train,y_train, cv=kfold)
print "Accuracy - Train CV: ", results.mean()
print "Accuracy - Train : ", metrics.accuracy_score(random_search.best_estimator_.predict(X_train), y_train)
print "Accuracy - Test : ", metrics.accuracy_score(random_search.best_estimator_.predict(X_test), y_test)
from bayes_opt import BayesianOptimization
from sklearn.cross_validation import cross_val_score

def rfccv(n_estimators, min_samples_split, max_features):
    return cross_val_score(RandomForestClassifier(n_estimators=int(n_estimators),
                                                  min_samples_split=int(min_samples_split),
                                                  max_features=min(max_features, 0.999),
                                                  random_state=2017),
                                                  X_train, y_train, 'f1', cv=kfold).mean()

gp_params = {"alpha": 1e5}

rfcBO = BayesianOptimization(rfccv, {'n_estimators': (100, 1000),
                                     'min_samples_split': (2, 25),
                                     'max_features': (0.1, 0.999)})

rfcBO.maximize(n_iter=10, **gp_params)

print('RFC: %f' % rfcBO.res['max']['max_val'])<jupyter_output>[31mInitialization[0m
[94m-------------------------------------------------------------------------------------[0m
 Step |   Time |      Value |   max_features |   min_samples_split |   n_estimators | 
    1 | 00m13s | [35m   0.59033[0m | [32m        0.1628[0m | [32m             2.7911[0m | [32m      891.4580[0m | 
    2 | 00m08s |    0.57056 |         0.1725 |              4.1269 |       543.9055 | 
    3 | 00m04s | [35m   0.61064[0m | [32m        0.7927[0m | [32m            21.6962[0m | [32m      275.7203[0m | 
    4 | 00m06s |    0.58312 |         0.2228 |              6.4325 |       437.6023 | 
    5 | 00m03s | [35m   0.61265[0m | [32m        0.3626[0m | [32m            12.2393[0m | [32m      236.6017[0m | 
[31mBayesian Optimization[0m
[94m-------------------------------------------------------------------------------------[0m
 Step |   Time |      Value |   max_features |   min_samples_split |   n_estimators | 
    6 | 00m17s | [35m   0.61354[0m |[...]
{'fraction_non_alphanumeric': 0.12551245663828445, 'fraction_numerical': 0.0499842321034374, 'mean_word_length': 3.200662251655629, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '2085', 'license': 'MIT', 'n_tokens_mistral': 2212, 'n_tokens_neox': 1961, 'n_words': 493, 'path': 'jupyter_notebooks/machine_learning/ebook_mastering_ml_in_6_steps/Chapter_4_Code/Code/Hyper_Parameter_Tuning.ipynb', 'repo_name': 'manual123/Nacho-Jupyter-Notebooks', 'size': '6342'}