<jupyter_start><jupyter_text>House Prices Advanced Regression Techniques_create by [<NAME>](https://www.kaggle.com/austinpoor)_* [Kaggle Competition](https://www.kaggle.com/c/house-prices-advanced-regression-techniques/data?select=train.csv)* [Github Repo](https://github.com/a-poor/kaggle-house-prices/tree/main)<jupyter_code>import sys
from pathlib import Path
DATA_DIR = Path("../input")
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set()
df_train = pd.read_csv(DATA_DIR / "train.csv")
df_train.head()
def clean_df(df:pd.DataFrame) -> pd.DataFrame:
    category_columns = ['MSSubClass',
        'MSZoning','Street','Alley','LotShape','LandContour',
        'Utilities','LotConfig','LandSlope','Neighborhood',
        'Condition1','Condition2','BldgType','HouseStyle',
        'OverallQual','OverallCond','RoofStyle','RoofMatl',
        'Exterior1st','Exterior2nd','MasVnrType','ExterQual',
        'ExterCond','Foundation','BsmtQual','BsmtCond',
        'BsmtExposure','BsmtFinType2','Heating','HeatingQC',
        'CentralAir','Electrical','KitchenQual','Functional',
        'FireplaceQu','GarageType','GarageFinish','GarageQual',
        'GarageCond','PavedDrive','PoolQC','Fence','MiscFeature',
        'SaleType','SaleCondition']
    for c in category_columns:
        df[c] = pd.Categorical(df[c])
    return df
df_train = clean_df(df_train)
df_train.sample(10)
df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1460 entries, 0 to 1459
Data columns (total 81 columns):
Id               1460 non-null int64
MSSubClass       1460 non-null category
MSZoning         1460 non-null category
LotFrontage      1201 non-null float64
LotArea          1460 non-null int64
Street           1460 non-null category
Alley            91 non-null category
LotShape         1460 non-null category
LandContour      1460 non-null category
Utilities        1460 non-null category
LotConfig        1460 non-null category
LandSlope        1460 non-null category
Neighborhood     1460 non-null category
Condition1       1460 non-null category
Condition2       1460 non-null category
BldgType         1460 non-null category
HouseStyle       1460 non-null category
OverallQual      1460 non-null category
OverallCond      1460 non-null category
YearBuilt        1460 non-null int64
YearRemodAdd     1460 non-null int64
RoofStyle        1460 non-null category
RoofMatl         1460 non-nu[...]<jupyter_text>__Sort Columns by $|CORR(column)|$__<jupyter_code>sales_corr = df_train.corr()["SalePrice"]\
    .reset_index()
sales_corr.columns = ["colname","sales_corr"]
sales_corr["abs_corr"] = np.abs(sales_corr["sales_corr"])
sales_corr = sales_corr.set_index("colname")
sales_corr = sales_corr.join(colvariance)
sales_corr.sort_values(
    ["abs_corr","col_variance"],
    ascending=False
)<jupyter_output><empty_output><jupyter_text>__Sort Columns by $Var(column)$__<jupyter_code>sales_corr.sort_values(
    ["col_variance","abs_corr"],
    ascending=False
)
from sklearn import impute
from sklearn import preprocessing
from sklearn import feature_extraction
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import linear_model
from sklearn import tree
from sklearn import ensemble
from xgboost import XGBRegressor
df = pd.get_dummies(df_train,drop_first=True)
X, y = df.drop("SalePrice",1), df["SalePrice"].values

imp = impute.SimpleImputer(strategy="median")
X = imp.fit_transform(X)

m = linear_model.LinearRegression().fit(X,y)
m.score(X,y)
def kfold_index_gen(N: int, k: int):
    w = N / k
    fold_info = pd.DataFrame({
        "indx": np.arange(N)
        })\
        .sample(frac=1.)\
        .reset_index(drop=True)
    fold_info["fold"] = (np.arange(N) // w).astype(int)
    for fold in range(k):
        train = fold_info[fold_info.fold != fold].indx
        test  = fold_info[fold_info.fold == fold].indx
        yield train.values, test.values

def kfold_data(X, y, k:int):
    N = len(X)
    for train_i, test_i in kfold_index_gen(N,k):
        yield X[train_i], X[test_i], y[train_i], y[test_i]
X_train, X_test, y_train, y_test = model_selection.train_test_split(X,y)
X_train.shape, X_test.shape, y_train.shape, y_test.shape
X_train.shape, y_train.shape, X_val.shape, y_val.shape
# Set parameters
models = {
    "Linear Regression": linear_model.LinearRegression(),
    "LASSO Regression": linear_model.Lasso(),
    "Decision Tree": tree.DecisionTreeRegressor(),
    "Random Forrest": ensemble.RandomForestRegressor(),
    "Gradient Boosted": ensemble.GradientBoostingRegressor(),
    "XGBoost": XGBRegressor(),
}
K = 8

# Split into Train/Test
X_train, X_test, y_train, y_test = model_selection.train_test_split(X,y)

# Run the loop
print(f"Training {len(models)} models on {K} folds")
scores = {}
for mname, m in models.items():
    print("Training model:", mname)
    scores[mname] = []
    for k, (X_train, X_val, y_train, y_val) in enumerate(kfold_data(X_train,y_train,K)):
        m.fit(X_train,y_train)
        s = m.score(X_val,y_val)
        scores[mname].append(s)
        print(f"[{mname:17} | {k}] {s:.4f}")
    print()

print("Results..")
for mname, m in models.items():
    print(f"[{mname:17}] μ={np.mean(scores[mname]):.4f} σ={np.std(scores[mname]):.4f}")
import itertools as it
results = pd.DataFrame(it.chain(*[[{"model":k,"score":n} for n in v] for k, v in scores.items()]))
results.head()
fig, ax = plt.subplots(figsize=(12,6))
ax = sns.barplot(
    x="score",
    y="model",
    data=results,
    orient="h",
    # ci="sd",
)
plt.title("Visualizing Naive Model Performance")
plt.xlabel("Validation Score (Error bar = STD)")
plt.ylabel("");
model_selection.GridSearchCV?<jupyter_output>[0;31mInit signature:[0m
[0mmodel_selection[0m[0;34m.[0m[0mGridSearchCV[0m[0;34m([0m[0;34m[0m
[0;34m[0m    [0mestimator[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mparam_grid[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0;34m*[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mscoring[0m[0;34m=[0m[0;32mNone[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mn_jobs[0m[0;34m=[0m[0;32mNone[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0miid[0m[0;34m=[0m[0;34m'deprecated'[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mrefit[0m[0;34m=[0m[0;32mTrue[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mcv[0m[0;34m=[0m[0;32mNone[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mverbose[0m[0;34m=[0m[0;36m0[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mpre_dispatch[0m[0;34m=[0m[0;34m'2*n_jobs'[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0merror_score[0m[0;34m=[0m[0mnan[0m[0;34m,[0m[0;34m[0m
[0;34m[0m    [0mreturn_train_score[0m[0;34m=[0m[0;32mFalse[0m[0;34m,[...]<jupyter_text>Perform Grid Search on Some of the Models<jupyter_code>import time
models = {}
param_grids = {}
X_train, X_test, y_train, y_test = model_selection.train_test_split(X,y)<jupyter_output><empty_output><jupyter_text>LASSO Regression<jupyter_code># Set the Parameters
model_name = "lasso"
param_grids[model_name] = {
    "alpha":np.linspace(0.001,5,10),
    "max_iter":10 ** np.arange(2,8,2)
}

# Build GridSearchCV Model
models[model_name] = model_selection.GridSearchCV(
    estimator=linear_model.Lasso(),
    param_grid=param_grids["lasso"],
    scoring="r2",
    n_jobs=-1,
    cv=5,
    refit=True,
    verbose=10
)

# Fit the Model
models[model_name].fit(X_train,y_train)

# Print the results
time.sleep(0.1)
print()
print(f"\"{model_name:>10s}\" => BEST SCORE: {models[model_name].best_score_:.4f}")<jupyter_output>Fitting 5 folds for each of 30 candidates, totalling 150 fits
[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.
[Parallel(n_jobs=-1)]: Done   5 tasks      | elapsed:    3.1s
[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:   20.4s
[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:  5.5min
[Parallel(n_jobs=-1)]: Done  24 tasks      | elapsed:  6.1min
[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:  6.4min
[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:  7.0min
[Parallel(n_jobs=-1)]: Done  53 tasks      | elapsed:  7.2min
[Parallel(n_jobs=-1)]: Done  64 tasks      | elapsed:  7.4min
[Parallel(n_jobs=-1)]: Done  77 tasks      | elapsed:  7.8min
[Parallel(n_jobs=-1)]: Done  90 tasks      | elapsed:  8.1min
[Parallel(n_jobs=-1)]: Done 105 tasks      | elapsed:  8.2min
[Parallel(n_jobs=-1)]: Done 120 tasks      | elapsed:  8.3min
[Parallel(n_jobs=-1)]: Done 137 tasks      | elapsed:  8.3min

"     lasso" => BEST SCORE: 0.9003
[Parallel(n_jobs=-1)[...]<jupyter_text>Random Forrest<jupyter_code>model_name = "random_forest"
param_grids[model_name] = {
    "n_estimators": [10,50,100,500,1_000,5_000,10_000],
    "max_depth": [1,3,5,7,10,20,None],
    "min_samples_split":[1,2,5,10,20],
    "min_samples_leaf":[1,2,5],
    "max_features":["auto","sqrt","log2"],
    "max_leaf_nodes":[None,],
    "min_impurity_decrease":[0,0.1,0.25],
    "max_samples":[None,0.2,0.5,0.8,0.9,1.0],
    "max_samples":[0.5,0.8,1.0],
}
models[model_name] = model_selection.GridSearchCV(
    estimator=ensemble.RandomForestRegressor(n_jobs=1,verbose=False),
    param_grid=param_grids[model_name],
    scoring="r2",
    n_jobs=-1,
    cv=5,
    refit=True,
    verbose=10
)
models[model_name].fit(X_train,y_train)
time.sleep(0.1)
print()
print("\"{model_name:>17s}\"BEST SCORE: {models[model_name].best_score_:.4f}")<jupyter_output>Fitting 5 folds for each of 30 candidates, totalling 150 fits
[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.
[Parallel(n_jobs=-1)]: Done   5 tasks      | elapsed:    3.3s
[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:    3.4s
[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    3.5s
[Parallel(n_jobs=-1)]: Batch computation too fast (0.1955s.) Setting batch_size=2.
[Parallel(n_jobs=-1)]: Done  24 tasks      | elapsed:    3.6s
[Parallel(n_jobs=-1)]: Batch computation too fast (0.0452s.) Setting batch_size=4.<jupyter_text>XGBoost<jupyter_code>model_name = "xgbr"
param_grids[model_name] = {
    "n_estimators": 10 ** np.arange(6),
    "max_depth": [3,4,5,6,7,8,9,10],
    "learning_rate": np.linspace(1e-5,0.3,5),
    "subsample": np.linspace(0.5,1.0,5),
    "colsample_bytree": np.linspace(0.5,1.0,5),
    "lambda": np.linspace(0,1.0,5),
    "alpha": np.linspace(0,1.0,5),
}
models[model_name] = model_selection.GridSearchCV(
    estimator=XGBRegressor(n_jobs=1,silent=1),
    param_grid=param_grids[model_name],
    scoring="r2",
    n_jobs=-1,
    cv=5,
    refit=True,
    verbose=10
)
models[model_name].fit(X_train,y_train)
print("\"{model_name:>17s}\"BEST SCORE: {models[model_name].best_score_:.4f}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17083177918687056, 'fraction_numerical': 0.061077955986572174, 'mean_word_length': 4.735294117647059, 'pattern_counts': {'":': 27, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '16353', 'license': 'MIT', 'n_tokens_mistral': 4779, 'n_tokens_neox': 4305, 'n_words': 694, 'path': 'notebooks/apoor-house-prices.ipynb', 'repo_name': 'a-poor/kaggle-house-prices', 'size': '10729'}