<jupyter_start><jupyter_text>Split train/test set<jupyter_code>test_df = df.sample(n=9258,random_state=44).reset_index(drop=True)
train_df = df.drop(test_df.index).reset_index(drop=True)
print(train_df.shape)
print(test_df.shape)<jupyter_output>(33000, 27)
(9258, 27)<jupyter_text>K-fold cross validation<jupyter_code>from sklearn.model_selection import KFold

#Create a new column called kfold and fill it with -1
train_df['kfold'] = - 1
#Initiate the kfold class 
kf = KFold(n_splits=3,random_state=3107,shuffle=True)
#Fill the new kfold column
for fold, (train,val) in enumerate(kf.split(train_df)):
    train_df.loc[val,'kfold'] = fold
train_df['kfold'].value_counts()
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.metrics import r2_score,mean_squared_error
train_df<jupyter_output><empty_output><jupyter_text>Model building and evaluation<jupyter_code>def model_running(model_choose,fold,data):
    #Split dataframe
    df_train = data[data['kfold'] != fold].reset_index(drop=True)
    df_test = data[data['kfold'] == fold].reset_index(drop=True)
    
    #Split X_train,X_test,y_train,y_test
    X_train = df_train.drop(['kfold','Price (bil)','Price/m2 (mil)'],axis=1)
    X_test = df_test.drop(['kfold','Price (bil)','Price/m2 (mil)'],axis=1)

    y_train = df_train['Price (bil)']
    y_test = df_test['Price (bil)']
    
    #Fitting model
    model = model_choose
    model.fit(X_train,y_train)

    #Predictions
    y_pred = model.predict(X_test)

    #Model evaluation
    r2 = r2_score(y_test,y_pred)
    rmse = mean_squared_error(y_test,y_pred,squared=False)
    
    return r2,rmse<jupyter_output><empty_output><jupyter_text>Linear regression<jupyter_code>lr = LinearRegression()
lr_r2 = []
lr_rmse = []
for fold in range(3):
    r2,rmse = model_running(lr,fold,train_df)
    lr_r2.append(r2)
    lr_rmse.append(rmse)
print(lr_r2)
print(f'Mean r square: {np.mean(lr_r2)}')
print(lr_rmse)
print(f'Mean root square error: {np.mean(lr_rmse)}')<jupyter_output>[0.7256361744330846, 0.718289501263002, 0.7176932229172428]
Mean r square: 0.7205396328711098
[2.6015409633699655, 2.57906438971359, 2.5927977280164267]
Mean root square error: 2.591134360366661<jupyter_text>Random forest regressor<jupyter_code>rf = RandomForestRegressor(random_state=3107)
rf_r2 = []
rf_rmse = []
for fold in range(3):
    r2,rmse = model_running(rf,fold,train_df)
    rf_r2.append(r2)
    rf_rmse.append(rmse)
print(rf_r2)
print(f'Mean r square: {np.mean(rf_r2)}')
print(rf_rmse)
print(f'Mean root square error: {np.mean(rf_rmse)}')<jupyter_output>[0.9750935074653052, 0.9761837130740468, 0.9724492707759747]
Mean r square: 0.9745754971051088
[0.7838329749671552, 0.7498902304563339, 0.8099804462316451]
Mean root square error: 0.7812345505517113<jupyter_text>XG boost<jupyter_code>xgb = XGBRegressor(random_state=3107)
xgb_r2 = []
xgb_rmse = []
for fold in range(3):
    r2,rmse = model_running(xgb,fold,train_df)
    xgb_r2.append(r2)
    xgb_rmse.append(rmse)
print(xgb_r2)
print(f'Mean r square: {np.mean(xgb_r2)}')
print(xgb_rmse)
print(f'Mean root square error: {np.mean(xgb_rmse)}')<jupyter_output>[0.9664563033695451, 0.9656677359844, 0.9643695741027349]
Mean r square: 0.9654978711522267
[0.9096466082876796, 0.9003510814422001, 0.9211248361252901]
Mean root square error: 0.9103741752850566<jupyter_text>Light GBM<jupyter_code>lgbm = LGBMRegressor(random_state=3107)
lgbm_r2 = []
lgbm_rmse = []
for fold in range(3):
    r2,rmse = model_running(lgbm,fold,train_df)
    lgbm_r2.append(r2)
    lgbm_rmse.append(rmse)
print(lgbm_r2)
print(f'Mean r square: {np.mean(lgbm_r2)}')
print(lgbm_rmse)
print(f'Mean root square error: {np.mean(lgbm_rmse)}')<jupyter_output>[0.9501526898193905, 0.9497112485342082, 0.9469428799879344]
Mean r square: 0.9489356061138444
[1.1088890829127558, 1.089672762646581, 1.1240349669819942]
Mean root square error: 1.1075322708471103<jupyter_text>- In 4 models, the best performance is Random Forest. But I will choose the Light GBM model due to the speed of processing to tuning hyperparameters.  Model tuning<jupyter_code>from sklearn.model_selection import RandomizedSearchCV

X = train_df.drop(['kfold','Price (bil)','Price/m2 (mil)'],axis=1)
y = train_df['Price (bil)']
#Max number of bins that feature values will be bucketed in
max_bin = [300,400,500]
#Shrinkage rate
learning_rate = [0.001,0.005,0.01,0.05]
#Number of iterations
num_iterations = [200,500,700,1000,1500,2000]
#Max number of leaves in one tree
num_leaves = [30,50,70,100]
#Boosting type
boosting_type = ['gbdt','dart']
# Create the random grid
random_grid = {'max_bin': max_bin,
               'learning_rate': learning_rate,
               'num_iterations': num_iterations,
               'num_leaves': num_leaves,
               'boosting_type': boosting_type
               }
print(random_grid)
# Use the random grid to search for best hyperparameters
# First create the base model to tune
lgbm = LGBMRegressor(random_state=3107)
# Random search of parameters, using 3 fold cross validation, 
# search across 100 different combinations, and use all available cores
lgbm_random = RandomizedSearchCV(estimator = lgbm, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=3107, n_jobs = -1)
# Fit the random search model
lgbm_random.fit(X, y)
lgbm_random.best_params_
#Fit tuned model
lgbm = LGBMRegressor(random_state=3107,num_leaves=30,num_iterations=2000,max_bin=500,learning_rate=0.05,boosting_type='dart')
lgbm_r2 = []
lgbm_rmse = []
for fold in range(3):
    r2,rmse = model_running(lgbm,fold,train_df)
    lgbm_r2.append(r2)
    lgbm_rmse.append(rmse)

print(lgbm_r2)
print(f'Mean r square: {np.mean(lgbm_r2)}')
print(lgbm_rmse)
print(f'Mean root square error: {np.mean(lgbm_rmse)}')<jupyter_output>[0.9725039792546691, 0.9726847504735819, 0.9721463142619764]
Mean r square: 0.9724450146634092
[0.8235731267481914, 0.8030882186737046, 0.8144216706510427]
Mean root square error: 0.8136943386909795<jupyter_text>Final model and evaluation<jupyter_code>final_model = LGBMRegressor(random_state=3107,num_leaves=30,num_iterations=2000,max_bin=500,learning_rate=0.05,boosting_type='dart')
final_model.fit(X, y)

import joblib
joblib.dump(final_model,'final_model.sav')
#Final evaluation
final_X_test = test_df.drop(['Price (bil)','Price/m2 (mil)'],axis=1)
final_y_test = test_df['Price (bil)']
pred = final_model.predict(final_X_test)
print('r2:',r2_score(final_y_test,pred))
print('rmse:',mean_squared_error(final_y_test,pred,squared=False))<jupyter_output>r2: 0.9571830133393139
rmse: 0.9609154004436522<jupyter_text>For reference: Tuning the Random Forest model - Noted: This process will make the very long times<jupyter_code># # Use the random grid to search for best hyperparameters
# # First create the base model to tune
# rf = RandomForestRegressor(random_state=3107)
# # Random search of parameters, using 3 fold cross validation, 
# # search across 100 different combinations, and use all available cores
# rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=3107, n_jobs = -1)
# # Fit the random search model
# rf_random.fit(X, y)

# #Bootstrap 
# bootstrap = [True, False]
# #The maximum depth of the tree
# max_depth = [10, 30, 50, 100, None]
# #The number of features to consider when looking for the best split
# max_features = ['auto', 'sqrt']
# #The minimum number of samples required to be at a leaf node
# min_samples_leaf = [1, 2, 4]
# #The minimum number of samples required to split an internal node
# min_samples_split = [2, 5, 10]
# #Number of trees in random forest
# n_estimators = [200, 500, 1000, 1500, 2000]

# # Create the random grid
# random_grid = {'bootstrap': bootstrap,
#                'max_depth': max_depth,
#                'max_features': max_features,
#                'min_samples_leaf': min_samples_leaf,
#                'min_samples_split': min_samples_split,
#                'n_estimators':n_estimators
#                }
# print(random_grid)

# rf_random.best_params_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11799193843898864, 'fraction_numerical': 0.116037620618053, 'mean_word_length': 5.806317539484621, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '434726', 'license': 'MIT', 'n_tokens_mistral': 3772, 'n_tokens_neox': 3149, 'n_words': 638, 'path': 'model.ipynb', 'repo_name': '123olala/House-Price-HCM-ver2', 'size': '8187'}