<jupyter_start><jupyter_text>Import packages and data<jupyter_code>import pandas as pd 
import numpy as np 

from sklearn.model_selection import train_test_split
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV

from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import mean_squared_error

RSEED = 42
df = pd.read_csv('data/train_modelling.csv')
df.head(2)<jupyter_output><empty_output><jupyter_text>Select columns and  Define X and y and  Train-test-split<jupyter_code># select columns for X
cols = ['mean_temp', 'mean_precip', 'mean_rel_humidity', 'mean_wind_dir', 'mean_wind_spd']
# define X and y
X = df[cols]
y = df['target']
# train-test-split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=RSEED)<jupyter_output><empty_output><jupyter_text>Linear Regression<jupyter_code># fitting the linear regression
lin_reg = LinearRegression()
lin_reg.fit(X_train, y_train)
# predicting and RMSE
y_pred_lin = lin_reg.predict(X_test)
np.sqrt(mean_squared_error(y_test, y_pred_lin))<jupyter_output><empty_output><jupyter_text>Decision Tree Regression<jupyter_code># Fitting a Decision Tree Regression
dtr =  DecisionTreeRegressor()
dtr.fit(X_train, y_train)
#predicting and RMSE
y_pred_dt = dtr.predict(X_test)
np.sqrt(mean_squared_error(y_test, y_pred_dt))<jupyter_output><empty_output><jupyter_text>Random Forest Regressor<jupyter_code># fitting a Random Forest Regression
rfr = RandomForestRegressor(random_state=RSEED)
rfr.fit(X_train, y_train)
# predicting and RMSE
y_pred_rfr = rfr.predict(X_test)
np.sqrt(mean_squared_error(y_test, y_pred_rfr))
# Return the coefficient of determination of the prediction (R-squared)
rfr.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Randomized Search CV<jupyter_code># import pprint (Data pretty printer) and use it to print current parameters of Random Forest
from pprint import pprint

print('Parameters currently in use:\n')
pprint(rfr.get_params())
# create a parameter grid for RandomSearchCV

# Number of trees in random forest
n_estimators = [int(x) for x in np.linspace(start=200, stop=2000, num=10)]
# No. of features to consider at every split
max_features = ['auto', 'sqrt']
# Maximum number of levels in tree
max_depth = [int(x) for x in np.linspace(10,110,num=11)]
max_depth.append(None)
# Minimum number of samples required to split a node
min_samples_split = [2,5,10]
# Minimum number of samples required at each leaf node
min_samples_leaf = [1,2,4]
# Method of selecting samples for training each tree
bootstrap = [True, False]

# Create the random grid
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}

pprint(random_grid)
# Use the random grid search for best hyperparameters
# First create the base model to tune
rf = RandomForestRegressor()
# Random search of parameters, using 3 fold cross validaition,
# search across 100 different combinations, and use all available cores
rf_random = RandomizedSearchCV(estimator=rf, param_distributions=random_grid, n_iter=100, cv=3, verbose=2, random_state=RSEED, n_jobs=-1)

# fit the random search model
rf_random.fit(X_train, y_train)
# best parameters after Search
rf_random.best_params_
# fitting Random Forest with best parameters
best_random = RandomForestRegressor(n_estimators=400, min_samples_split=2, min_samples_leaf=1,
                                    max_features='sqrt', max_depth=None, bootstrap=False, random_state=RSEED)
best_random.fit(X_train, y_train)
# predicting and RMSE
y_pred = best_random.predict(X_test)
np.sqrt(mean_squared_error(y_test, y_pred)).round(2)
# Evaluate Random Search with accuracy and RMSE comparison

def evaluate (model, test_features, test_labels) :
    predictions = model.predict(test_features)
    errors = abs(predictions - test_labels)
    mape = 100 * np.mean(errors / test_labels)
    accuracy = 100 - mape
    print('Model Performance')
    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
    print('Accuracy = {:0.2f}%.'.format(accuracy))

    return accuracy

base_model = RandomForestRegressor(random_state=RSEED)
base_model.fit(X_train, y_train)
base_accuracy = evaluate(base_model, X_test, y_test)
y_base = base_model.predict(X_test)
base_rmse = np.sqrt(mean_squared_error(y_test, y_base)).round(2)
print(base_rmse)

best_random = rf_random.best_estimator_
random_accuracy = evaluate(best_random, X_test, y_test)
y_random = best_random.predict(X_test)
random_rmse = np.sqrt(mean_squared_error(y_test, y_random)).round(2)
print(random_rmse)
print('Improvement in Accuracy of {:0.2f}%.'.format(100 * (random_accuracy - base_accuracy) / base_accuracy))
print('Improvement in RMSE of {:0.2f}%.'.format(abs(100*(random_rmse-base_rmse)/base_rmse)))
print('Improvement in RMSE of {:0.2f} total.'.format(abs(random_rmse-base_rmse)))<jupyter_output>Model Performance
Average Error: 16.9689 degrees.
Accuracy = 65.01%.
27.92
Model Performance
Average Error: 15.9977 degrees.
Accuracy = 68.55%.
27.35
Improvement in Accuracy of 5.45%.
Improvement in RMSE of 2.04%.
Improvement in RMSE of 0.57 total.<jupyter_text>GridSearchCV<jupyter_code># again the best parameters after RandomSearchCV
rf_random.best_params_
# create the parameter grid based on the results of random search

param_grid = {
    'bootstrap': [False],
    'max_depth': [80, 90, 100, 110, None],
    'max_features': [2, 3, 'sqrt'],
    'min_samples_leaf': [1, 2],
    'min_samples_split': [2, 4],
    'n_estimators': [100, 200, 300, 400, 1000]
}

rf = RandomForestRegressor()
grid_search = GridSearchCV(estimator=rf, param_grid=param_grid, cv=3, n_jobs=-1, verbose=2)
# Fit the grid search to the data
grid_search.fit(X_train, y_train)

grid_search.best_params_
# fitting Random Forest with best parameters (here: the same as before)
best_grid = RandomForestRegressor(n_estimators=400, min_samples_split=2, min_samples_leaf=1,
                                    max_features='sqrt', max_depth=None, bootstrap=False, random_state=RSEED)
best_grid.fit(X_train, y_train)
# predicting and RMSE
y_pred = best_random.predict(X_test)
np.sqrt(mean_squared_error(y_test, y_pred)).round(2)
# Evaluate GridSearchCV
best_grid = grid_search.best_estimator_
grid_accuracy = evaluate(best_grid, X_test, y_test)

print('Improvement of {:0.2f}%.'.format(100*(grid_accuracy-base_accuracy)/base_accuracy))<jupyter_output>Model Performance
Average Error: 16.0313 degrees.
Accuracy = 68.40%.
Improvement of 5.22%.
{'fraction_non_alphanumeric': 0.10206489675516224, 'fraction_numerical': 0.024336283185840708, 'mean_word_length': 5.898270600203459, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '65864', 'license': 'MIT', 'n_tokens_mistral': 2497, 'n_tokens_neox': 2317, 'n_words': 565, 'path': 'Random_Forest_Search.ipynb', 'repo_name': 'JeromeSauer/Project2_AirQuality', 'size': '6780'}