<jupyter_start><jupyter_text>Model Experimentation1. Random Forest Regressor2. XGBoost Regressor3. LightGBM Regressor  Import Libraries<jupyter_code># dataframe packages
import pandas as pd
import numpy as np
from skopt.space import Categorical, Integer, Real
from skopt.utils import use_named_args
from skopt import gp_minimize

# statistical packages
import math
from scipy.stats import uniform
from math import sqrt

# modeling packages
from sklearn.ensemble import RandomForestRegressor
import lightgbm as lgb
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
from sklearn.svm import SVR

# evaluation packages
from sklearn.metrics import r2_score,mean_squared_error, mean_squared_log_error
from sklearn.model_selection import cross_val_score, RepeatedKFold, train_test_split, RandomizedSearchCV, GridSearchCV
from sklearn.model_selection import TimeSeriesSplit

# scaling packages
from sklearn.preprocessing import StandardScaler, MinMaxScaler

# visualisation packages
import seaborn as sns
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Load Dataset<jupyter_code>directory = 'C:/Users/chery/Documents/NUS Y4 Sem 2 Modules/BT4222/Dataset/Primary Dataset/modelling_dataset.csv'
model_df = pd.read_csv(directory)
model_df.head()<jupyter_output><empty_output><jupyter_text>Train Test Split1. Training set 0.62. Validation set 0.23. Test set - 0.2The output variable will be Unit Price ($ PSM)<jupyter_code>X = model_df.drop(["Unit Price ($ PSM)"], axis=1)
y = model_df['Unit Price ($ PSM)']

print('Shape of X is:', X.shape)
print('Shape of Y is:', y.shape)<jupyter_output>Shape of X is: (54674, 52)
Shape of Y is: (54674,)<jupyter_text>Train Test Split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)

print('Shape of X_train is:', X_train.shape)
print('Shape of y_train is:', y_train.shape)
print('Shape of X_test is:', X_test.shape)
print('Shape of y_test is:', y_test.shape)<jupyter_output>Shape of X_train is: (38271, 52)
Shape of y_train is: (38271,)
Shape of X_test is: (16403, 52)
Shape of y_test is: (16403,)<jupyter_text>Scaling<jupyter_code>all_features = list(X_train.columns)

standardScale_vars = ['Area (SQM)',
                      'Floor Number',
                      'PPI',
                      'Average Cases Per Year',
                      'Nearest Primary School',
                      'nearest_station_distance']

minMax_vars = ['Remaining Lease']

remaining_features = [x for x in all_features if x not in standardScale_vars and x not in minMax_vars]
s_scaler = StandardScaler()
mm_scaler = MinMaxScaler()

s_scaled = pd.DataFrame(s_scaler.fit_transform(X_train.loc[:, standardScale_vars].copy()), columns=standardScale_vars, index=X_train.index)
mm_scaled = pd.DataFrame(mm_scaler.fit_transform(X_train.loc[:, minMax_vars].copy()), columns=minMax_vars, index=X_train.index)

X_train = pd.concat([s_scaled, 
                     mm_scaled,
                     X_train.loc[:, remaining_features].copy()], axis=1)
X_train
s_scaled_test = pd.DataFrame(s_scaler.transform(X_test.loc[:, standardScale_vars].copy()), columns=standardScale_vars, index=X_test.index)
mm_scaled_test = pd.DataFrame(mm_scaler.fit_transform(X_test.loc[:, minMax_vars].copy()), columns=minMax_vars, index=X_test.index)

X_test = pd.concat([s_scaled_test, 
                     mm_scaled_test,
                     X_test.loc[:, remaining_features].copy()], axis=1)
X_test<jupyter_output><empty_output><jupyter_text>Model Tuning Split training set into training and evaluation<jupyter_code>X_train, X_eval, y_train, y_eval = train_test_split(X_train, y_train, test_size=0.2, random_state=42, shuffle=True)

print('Shape of X_train is:', X_train.shape)
print('Shape of y_train is:', y_train.shape)
print('Shape of X_eval is:', X_eval.shape)
print('Shape of y_eval is:', y_eval.shape)<jupyter_output>Shape of X_train is: (30616, 52)
Shape of y_train is: (30616,)
Shape of X_eval is: (7655, 52)
Shape of y_eval is: (7655,)<jupyter_text>Random Forest Regressor  Select hyperparameters<jupyter_code># Create the parameter grid: gbm_param_grid 
gbm_param_grid = {'max_depth':[i for i in range(30, 50)], 
                  'min_samples_leaf':[i for i in range(1, 20)], 
                  'min_samples_split':[i for i in range(2, 10)], 
                  'min_impurity_decrease':[i for i in range(10, 20)], 
                  'min_weight_fraction_leaf':[i/10.0 for i in range(0,5)],
                  'max_leaf_nodes':[i for i in range(10, 20)], 
}

# Instantiate the regressor: lgbm
model = RandomForestRegressor(bootstrap=False, criterion='mse', max_features='sqrt', 
                              n_estimators=6500, random_state=42) 

# Perform random search: grid_mse
clf = RandomizedSearchCV(model, gbm_param_grid, random_state=0)


# Fit randomized_mse to the data
search = clf.fit(X_train, y_train)

# Print the best parameters and lowest RMSE
from pprint import pprint
pprint(search.best_estimator_.get_params())
model = RandomForestRegressor(bootstrap=False,
                              criterion='mse',
                              max_depth=33,
                              max_features='sqrt',
                              min_samples_leaf=2,
                              min_samples_split=2,
                              n_estimators=6500,
                              random_state=42)
model.fit(X_train, y_train)
predict_price_xgb = model.predict(X_eval)
predict_price_xgb<jupyter_output><empty_output><jupyter_text>Both y_test (the actual test values) and predict_price (the predicted values) are reshaped in order to compare the model accuracy using the root mean squared error.<jupyter_code>y_eval_xgb = np.array(y_eval)
y_eval_xgb = y_eval_xgb.reshape(-1,1)
predict_price_xgb = predict_price_xgb.reshape(-1,1)<jupyter_output><empty_output><jupyter_text>Root Mean Squared Error (RMSE)<jupyter_code>eval_mse = mean_squared_error(y_eval_xgb, predict_price_xgb)
rmse = sqrt(eval_mse)
print('RMSE: %f' % rmse)<jupyter_output>RMSE: 850.815577<jupyter_text>Let’s compare this to the mean value across the test data:<jupyter_code>np.mean(y_eval_xgb)
print('Size of error is approximately:', float(rmse/np.mean(y_eval_xgb)*100))<jupyter_output>Size of error is approximately: 7.586371582836665<jupyter_text>Mean Absolute Percentage Error<jupyter_code>def mean_absolute_percentage_error(y_true, y_pred): 
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

mape_xgb = mean_absolute_percentage_error(y_eval_xgb, predict_price_xgb)
print('MAPE:', mape_xgb)<jupyter_output>MAPE: 4.762758307130743<jupyter_text>Root Mean Squared Log Error<jupyter_code>eval_msle = mean_squared_log_error(y_eval_xgb, predict_price_xgb)
rmsle = sqrt(eval_msle)
print('RMSLE: %f' % rmsle)<jupyter_output>RMSLE: 0.065679<jupyter_text>Average house price will be 1.05 times as big as the true value  Adjusted R^2<jupyter_code>adj_r = 1 - (1-model.score(X_train, y_train))*(len(y_train)-1)/(len(y_train)-X_train.shape[1]-1)
print('Adjusted R^2: %f' % adj_r)<jupyter_output>Adjusted R^2: 0.989879<jupyter_text>Predict on Test Dataset<jupyter_code>predict_price = model.predict(X_test)
y_test_xgb = np.array(y_test)
y_test_xgb = y_test_xgb.reshape(-1,1)
predict_test_price = predict_price.reshape(-1,1)<jupyter_output><empty_output><jupyter_text>RMSE<jupyter_code>test_mse = mean_squared_error(y_test_xgb, predict_test_price)
rmse = sqrt(test_mse)
print('RMSE: %f' % rmse)
np.mean(y_test_xgb)
print('Size of error is approximately:', float(rmse/np.mean(y_test_xgb)*100))<jupyter_output>Size of error is approximately: 7.93971087057534<jupyter_text>MAPE<jupyter_code>mape_test_xgb = mean_absolute_percentage_error(y_test_xgb, predict_test_price)
print('MAPE:', mape_test_xgb)<jupyter_output>MAPE: 4.853895316690389<jupyter_text>RMSLE<jupyter_code>test_msle = mean_squared_log_error(y_test_xgb, predict_test_price)
rmsle = sqrt(test_msle)
print('RMSLE: %f' % rmsle)
# Feature Importance
imp = pd.Series(data= model.feature_importances_, index= X_train.columns).sort_values(ascending=False)
plt.figure(figsize=(10,12))
plt.title("Feature importance")
ax = sns.barplot(y=imp.index, x=imp.values, palette="Blues_d", orient='h')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10718152291591483, 'fraction_numerical': 0.02959220498015157, 'mean_word_length': 5.0509461426492, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '253981', 'license': 'MIT', 'n_tokens_mistral': 3227, 'n_tokens_neox': 2985, 'n_words': 612, 'path': 'Model Experimentation/Model Experimentation - Random Forest Regressor.ipynb', 'repo_name': 'SG-Property-Valuation-Model/Modelling', 'size': '8313'}