<jupyter_start><jupyter_text>Model Tuning<jupyter_code>%config IPCompleter.greedy=True
import numpy as np
import pandas as pd
import sys
import inspect

#Add the scripts directory to the sys path
sys.path.append("../src/data")
sys.path.append("../src/features")

from make_dataset import get_data
from data_processor import DataProcessor
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
import xgboost as xgb
from sklearn.metrics import mean_absolute_error
# Show all rows and columns in the display
pd.set_option("display.max_columns", None)
pd.set_option("display.max_rows", None)
import warnings
warnings.filterwarnings('ignore')
#get the train data
X_train, y_train = get_data(data_string="train")
#get the test data
X_test, y_test = get_data(data_string="test")<jupyter_output><empty_output><jupyter_text>Baseline Model  Lets create a baseline model. In this case, let's say our predicted value is median of y_train<jupyter_code>y_pred = pd.Series(np.zeros(len(y_test)))
y_pred[:] = y_train.median()
mean_absolute_error(y_test, y_pred)<jupyter_output><empty_output><jupyter_text>Baseline model MAE is 0.0697  Other Models There are more than 500 columns and hence trying different models such as Random Forest, Gradient Boosting, XGBoost will take a lot of time. Hence, lets fix with XGBoost model and hypertune the parameters.  1. Linear Regression<jupyter_code>dp = DataProcessor(cols_to_remove=["parcelid", "propertyzoningdesc", "rawcensustractandblock", "regionidneighborhood", "regionidzip", "censustractandblock"], 
                  datecol="transactiondate")

lin_reg = LinearRegression(n_jobs=-1)

pipeline = Pipeline([
    ("dataprocessor", dp),
    ("lin_reg", lin_reg)
])
    
pipeline.fit(X_train, y_train)
y_pred = pipeline.predict(X_test)
print("mean absolute error: {0:.4f}".format(mean_absolute_error(y_test, y_pred)))<jupyter_output>mean absolute error: 0.2805<jupyter_text>Linear Regression MAE is worse than our baseline model.   2. XGBoost<jupyter_code>dp = DataProcessor(cols_to_remove=["parcelid", "propertyzoningdesc", "rawcensustractandblock", "regionidneighborhood", "regionidzip", "censustractandblock"], 
                  datecol="transactiondate")

xgb_reg = xgb.sklearn.XGBRegressor(learning_rate=0.1, n_estimators = 100, objective='reg:squarederror', 
                                    eval_metric="mae", random_state = 42, verbosity=1, n_thread=-1)

pipeline = Pipeline([
    ("dataprocessor", dp),
    ("xgb_reg", xgb_reg)
])
    
pipeline.fit(X_train, y_train)
y_pred = pipeline.predict(X_test)
print("mean absolute error: {0:.4f}".format(mean_absolute_error(y_test, y_pred)))<jupyter_output>mean absolute error: 0.0895<jupyter_text>Gradient Boosting has the best score among all three models..  XGBoost is generally better over Gradient Boosting Model as XGBoost is a regualarized model that controls over fitting and in addition XGBoost is also better in terms of speed and memory utilization Lets find the best XGBoost model using hyperparameter tuning<jupyter_code>def xgb_gridsearch(param_test, xgb_reg):
    
    dp = DataProcessor(cols_to_remove=["parcelid", "propertyzoningdesc", "rawcensustractandblock", "regionidneighborhood", "regionidzip", "censustractandblock"], 
                      datecol="transactiondate")

    pipeline = Pipeline([
        ("dataprocessor", dp),
        ("xgb_reg", xgb_reg)
    ])

    grid_search = GridSearchCV(estimator=pipeline, param_grid=param_test, n_jobs=-1, cv=3, verbose=1)

    grid_search.fit(X_train, y_train)

    print("----- Grid Search cv results ----- \n")
    for mean_score, params in zip(grid_search.cv_results_["mean_test_score"], grid_search.cv_results_["params"]):
        print(-(mean_score), params)

    print("\n----- Grid Search best parameters ------ \n", grid_search.best_params_)
    print("\n")
    print("----- Grid Search best score ------ \n", -(grid_search.best_score_))<jupyter_output><empty_output><jupyter_text>Step 1: Fix learning_rate and n_estimators<jupyter_code>param_test = {
    'xgb_reg__learning_rate': [0.1, 0.2, 0.3]
}

xgb_reg = xgb.sklearn.XGBRegressor(learning_rate=0.1, n_estimators=1000, max_depth=5, 
                                    min_child_weight=1, gamma=0, max_delta_step=0, 
                                    subsample=0.8, colsample_bytree=0.8, colsample_bylevel=1, 
                                    colsample_bynode=1, reg_lambda=1, reg_alpha=0, 
                                    scale_pos_weight=1, missing=None, objective='reg:squarederror', 
                                    eval_metric='mae', seed=0, booster='gbtree')# , silent=0, nthread=-1)

xgb_gridsearch(param_test, xgb_reg)<jupyter_output>Fitting 3 folds for each of 3 candidates, totalling 9 fits
{'fraction_non_alphanumeric': 0.10626375825495298, 'fraction_numerical': 0.01380828497098259, 'mean_word_length': 4.420824295010846, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '255864', 'license': 'MIT', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1605, 'n_words': 387, 'path': 'notebooks/.ipynb_checkpoints/05-fs-model-tuning-checkpoint.ipynb', 'repo_name': 'firozgit/Zillow-Home-Value-Prediction', 'size': '4997'}