<jupyter_start><jupyter_text>Objectives:1. Learn important feature selection- SelectKBest from scikit learn- Correlation matrix- Feature importance from classifiers Train preparation:1. Delete certain Nan cols2. Keep only selected numeric & categorical features  Packages<jupyter_code>!pip install catboost
!pip install category_encoders
# My custom pypi.org package
!pip install --upgrade pianko
from pianko.nan_statistics import nan_statistics
from pianko.corr_filter import corr_filter
from pianko.build_pipe import build_pipe
from pianko.first_tune import first_tune
from pianko.fine_tune import fine_tune

from pianko.plotting import plot_learning_curve

from pianko.transformers import CatEncoder
from pianko.transformers import ColumnKeeper
from pianko.transformers import NanNumFiller
from pianko.transformers import NanCatFiller
from pianko.transformers import NanRemover
from pianko.transformers import LogTransformer
from pianko.transformers import QuantileRemover
from pianko.transformers import IQRRemover

from pianko.FeatureTransformer import FeaturesTransformers
import pandas as pd
import numpy as np
from google.colab import drive

# Plots
import matplotlib.pyplot as plt
import seaborn as sns

# Printing
import pprint

# Feature selection
from sklearn.feature_selection import chi2
from sklearn.feature_selection import f_classif
from sklearn.feature_selection import SelectKBest

# Encoding
from sklearn.preprocessing import OrdinalEncoder
import category_encoders as ce

# Pipeline
from sklearn.pipeline import Pipeline
from sklearn.impute import KNNImputer, SimpleImputer
from sklearn.pipeline import make_pipeline
from scipy.stats import skew

# Model
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from xgboost.sklearn import XGBRegressor
from catboost import CatBoostRegressor
from sklearn.linear_model import SGDRegressor
from sklearn.linear_model import ElasticNet
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from lightgbm import LGBMRegressor

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.preprocessing import OrdinalEncoder, StandardScaler
from sklearn.preprocessing import OneHotEncoder, MinMaxScaler, RobustScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import learning_curve

# For my functions
from sklearn.model_selection import cross_val_score
from sklearn.compose import ColumnTransformer

# Metrics
from sklearn.metrics import mean_squared_log_error
from sklearn.metrics import mean_squared_error

# Neural net
from keras.utils.vis_utils import plot_model
from keras.models import Sequential
from keras.layers import Dense, Input<jupyter_output>/usr/local/lib/python3.7/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
  import pandas.util.testing as tm<jupyter_text>Constants<jupyter_code>train_proportion = 0.9
validation_proportion = 0.1
rand_state = 42

# Feature selection
select_k_numerics = 30  # select 20 out of 36 numeric features
nan_drop_thresh = 0.25<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def get_order(df, feature):
    # Impose order depending on the median price
    subset = df[[feature, 'SalePrice']]
    s = subset.groupby([feature]).median().sort_values(by='SalePrice')
    return list(s.index)

def prepare_labels(ordered_features):
    return {k: ordered_features.index(k) + 1 for k in ordered_features}<jupyter_output><empty_output><jupyter_text>Data download<jupyter_code># Setting file download from google Drive
drive.mount('/content/gdrive')
gdrive_path = '/content/gdrive/MyDrive/'
df_path = 'ML/Datasets/2.HousePrices/'

train_path = gdrive_path + df_path + "train.csv"
test_path =  gdrive_path + df_path + "test.csv"
submission_path = gdrive_path + df_path + "submission.csv"
sample_submission_path = gdrive_path + df_path + "sample_submission.csv"
train = pd.read_csv(train_path, index_col='Id')
test = pd.read_csv(test_path, index_col='Id')
target = train['SalePrice']<jupyter_output><empty_output><jupyter_text>**Nan statistics & remove cols with high Nan percentage**<jupyter_code># Remove columns with nan % > threshold
res_nan_stat = nan_statistics(train, nan_thresh=0.2)
print(res_nan_stat)
NanRemover().transform(train, res_nan_stat.keys())
NanRemover().transform(test, res_nan_stat.keys())<jupyter_output>Col -- Nan percentage
PoolQC 99.52054794520548
MiscFeature 96.30136986301369
Alley 93.76712328767123
Fence 80.75342465753424
FireplaceQu 47.26027397260274
{'Alley': 93.76712328767123, 'FireplaceQu': 47.26027397260274, 'PoolQC': 99.52054794520548, 'Fence': 80.75342465753424, 'MiscFeature': 96.30136986301369}<jupyter_text>**Insight 1: Remove features with Nans > 40% (PoolQC, MiscFeature, Alley, Fence, FireplaceQu)**  **Fill Na**<jupyter_code>nan_statistics(train, nan_thresh=0.0).items()
cat_colnames = train.select_dtypes(include=['O']).columns
num_colnames = train.select_dtypes(include=['int64', 'float64']).columns
num_test_colnames = [c for c in num_colnames if not (c == 'SalePrice')]

NanCatFiller().transform(train, cat_colnames, val='Missing')
NanCatFiller().transform(test, cat_colnames, val='Missing')

NanNumFiller().transform(train, num_colnames, method='median')
NanNumFiller().transform(test, num_test_colnames, method='median')
nan_statistics(train, nan_thresh=0.0)
nan_statistics(test, nan_thresh=0.0)<jupyter_output>Col -- Nan percentage<jupyter_text>**Explore categorical**<jupyter_code>cat_features = train.select_dtypes(include=['O']).columns
print(len(cat_features))
def plot(feature):
    subset = train[[feature, 'SalePrice']]
    s = subset.groupby([feature]).median().sort_values(by='SalePrice')
    
    fig = plt.figure(figsize=(15, 5))
    plt.subplot(1, 2, 1)
    sns.histplot(x = feature, data=train, stat='percent')
    plt.subplot(1, 2, 2)
    sns.boxplot(x=feature, y='SalePrice', data=train, order=s.index)

plot(cat_features[0])
plot(cat_features[1])
plot(cat_features[2])
plot(cat_features[3])
plot(cat_features[4])
plot(cat_features[5])
plot(cat_features[6])
plot(cat_features[7])
plot(cat_features[8])
plot(cat_features[9])
plot(cat_features[10])
plot(cat_features[11])<jupyter_output><empty_output><jupyter_text>- MSZoning    - Order: I C FV A RH RM RL RP- LotShape    - R2 + R3 + R1 unite to 'irregular'- Utilities    - Remove column- Street    - Remove column- LotConfig    - Order: Inside FR2 Coner CullDSac FR3    - Remove FR3- LandSlope    - Mod & Sev -> unite    - Order: Gtl -> Other- Condition2    - Remove column- BidgeType    - Remove column  **Encode categorical**<jupyter_code>for f in cat_features:
    order = get_order(train, f)

    labels_dict = prepare_labels(order)
    labels_dict.update({'Missing': 0})

    CatEncoder().transform(train, f, labels_dict)
    CatEncoder().transform(test, f, labels_dict)

    train[f] = train[f].astype('int64')
    test[f] = test[f].astype('int64')
train.dtypes.unique(), test.dtypes.unique()
test.head()
test.head()
test['MSZoning'].value_counts()
train['MSZoning'].value_counts()<jupyter_output><empty_output><jupyter_text>**Feature Engineering**<jupyter_code>train.head()<jupyter_output><empty_output><jupyter_text>**If has garage feature**<jupyter_code>train['HasGarage'] = np.where(train['GarageArea'] != 0, 1, 0)
test['HasGarage'] = np.where(test['GarageArea'] != 0, 1, 0)<jupyter_output><empty_output><jupyter_text>**Has Mas Vnr**<jupyter_code>train['HasMasVnr'] = np.where(train['MasVnrArea'] != 0, 1, 0)
test['HasMasVnr'] = np.where(test['MasVnrArea'] != 0, 1, 0)<jupyter_output><empty_output><jupyter_text>**Skewness check, log Transform**<jupyter_code>sns.histplot(train['LotArea'])
skewed_features = (train.skew().sort_values(ascending=False)).index
skewed_features
LogTransformer().transform(train, skewed_features, offset=0.001)
LogTransformer().transform(test, skewed_features.drop('SalePrice'), offset=0.001)
sns.histplot(train['LotArea'])
sns.histplot(train['SalePrice'])
train.skew().sort_values(ascending=True)<jupyter_output><empty_output><jupyter_text>**IQR Filter**<jupyter_code>sns.histplot(train['SalePrice'])
#IQRRemover.transform(train, ['SalePrice'])
print(f'Size before: {train.shape}')
train = QuantileRemover().transform(train, ['SalePrice'], 0.02, 0.98)
print(f'Size after: {train.shape}')
sns.histplot(train['SalePrice'])
nan_statistics(train), nan_statistics(test)<jupyter_output>Col -- Nan percentage
Col -- Nan percentage<jupyter_text>**Select K best features**<jupyter_code># Choose numeric types and try SelectKBest features
target = train['SalePrice']
train.drop(columns=['SalePrice'], inplace=True)
features_selector = SelectKBest(score_func=f_classif, k=select_k_numerics)
fit = features_selector.fit(train, target)
feature_scores = pd.DataFrame({
    'Scores': pd.Series(fit.scores_),
    'Colnames': pd.Series(train.columns)
})
feature_scores = feature_scores.nlargest(select_k_numerics, 'Scores')
k_best_features = feature_scores['Colnames'].values
k_best_features<jupyter_output><empty_output><jupyter_text>**Heatmap for numerics**<jupyter_code>train['SalePrice'] = target
corrs = train.drop('SalePrice',axis=1).corrwith(train['SalePrice']).sort_values(ascending=False)

fig,axes=plt.subplots(1,1,figsize=(12,9))
axes.axhline(corrs[corrs>0].mean(), ls=':',color='black',linewidth=2)
axes.text(25.5,corrs[corrs>0].mean()+.015, "Average = {:.3f}".format(corrs[corrs>0].mean()),color='black',size=14)
axes.axhline(corrs[corrs<0].mean(), ls=':',color='black',linewidth=2)
axes.text(25.5,corrs[corrs<0].mean()-.035, "Average = {:.3f}".format(corrs[corrs<0].mean()),color='black',size=14)
sns.barplot(y=corrs,x=corrs.index,palette='Spectral')
plt.title('Correlation of Sale Price to other Features',size=20,color='black',y=1.03)
plt.xticks(rotation=90)
for p in axes.patches:
            value = p.get_height()
            if value <=.5:
                continue
            x = p.get_x() + p.get_width()-.9
            y = p.get_y() + p.get_height()+(.02*value)
            axes.text(x, y, str(value)[1:5], ha="left",fontsize=12,color='#000000')
plt.tight_layout()
corr_matrix = train.corr()

plt.figure(figsize=(15,15))
sns.heatmap(corr_matrix, cmap="RdYlGn")<jupyter_output><empty_output><jupyter_text>**Insight. There are ~14 numeric features, highly correlated with the target**<jupyter_code>corr_res = corr_filter(
    corr_matrix,
    target_feature='SalePrice',
    abs_thresh=0.4)
del corr_res['SalePrice']
best_fatures_corr = corr_res.keys()
set_k_best = set(k_best_features)
set_corr = set(best_fatures_corr)<jupyter_output><empty_output><jupyter_text>**Keep only relevant features**<jupyter_code>features_to_keep = list(set_k_best.union(set_corr))
print(features_to_keep)
print(len(features_to_keep), 'SalePrice' in features_to_keep)
ColumnKeeper().transform(train, features_to_keep + ['SalePrice'])
ColumnKeeper().transform(test, features_to_keep)
train.shape, test.shape
train.head()<jupyter_output><empty_output><jupyter_text>**Train Validation split (& Log transform of target)**<jupyter_code>y = train['SalePrice']
train.drop(columns=['SalePrice'], inplace=True)
X_train_val, X_test, y_train_val, y_test = train_test_split(
    train, 
    y,
    train_size=train_proportion,
    random_state=rand_state
)
X_train_val.shape, X_test.shape, y_train_val.shape, y_test.shape<jupyter_output><empty_output><jupyter_text>**Pipeline for features scaling + Transformers**<jupyter_code>num_features = list(train.select_dtypes(include=['int64', 'float64']).columns)
cat_features = list(train.select_dtypes(include=['O']).columns)

print(cat_features)
print(num_features)
num_pipeline = Pipeline(
    steps = [
             ('imputer', SimpleImputer()),
             ('scaler', RobustScaler())
    ]
)
num_transformer = FeaturesTransformers(num_features, num_pipeline)<jupyter_output><empty_output><jupyter_text>**Approximate tune, try models**<jupyter_code>models = [
          #('Regression', LinearRegression()),
          #('RF', RandomForestRegressor()),
          ('CatBoost', CatBoostRegressor(verbose=0)),
          ('XGB', XGBRegressor()),
          #('Elastic', ElasticNet()),
          #('SVR', SVR()),
          #('GradientBoost', GradientBoostingRegressor()),
          #('LGBM', LGBMRegressor())
          #('SGD', SGDRegressor())
]

res_dict = first_tune(
    models,
    scoring='neg_mean_squared_error',
    X_train=X_train_val,
    Y_train=y_train_val,
    transformers = dict(
        num=num_transformer
    )
)
d_sorted = {k:v.mean() for k, v in sorted(res_dict.items(), key = lambda item: item[1].mean())}
d_sorted
plot_learning_curve(
    CatBoostRegressor(verbose=0),
    X_train_val,
    y_train_val,
    train_sizes = [50, 100, 500, 1000],
    cv = 5,
    scoring = 'neg_mean_squared_error'
)<jupyter_output><empty_output><jupyter_text>**Cat Boost Regression shows best metrics**   **Model fine tune**<jupyter_code>catboost_models = [(
    'CatBoost',
    CatBoostRegressor(verbose=0),{
        'model__n_estimators': [400, 600, 800],
        'model__depth': [5, 6],
        'model__learning_rate': [0.03, 0.06],
        'model__l2_leaf_reg': [0.3, 0.5, 0.1]
    }
)]

catboost_res = fine_tune(
    catboost_models,
    X_train_val,
    y_train_val,
    strategy='GridSearchCV',
    search_params = dict(
        verbose=3,
        return_train_score=True,
        n_jobs=-1,
        scoring='neg_root_mean_squared_error',
        cv=6
    ),
    transformers = dict(
        numeric = num_transformer
    )
)<jupyter_output>{'verbose': 3, 'return_train_score': True, 'n_jobs': -1, 'scoring': 'neg_root_mean_squared_error', 'cv': 6}
Fitting 6 folds for each of 36 candidates, totalling 216 fits<jupyter_text>**Best Model Selection**<jupyter_code>best_model_score = catboost_res['CatBoost'].best_score_
best_estimator = catboost_res['CatBoost'].best_estimator_
print(best_model_score)
best_estimator['model']
plot_learning_curve(
    best_estimator,
    X_train_val,
    y_train_val,
    train_sizes = [50, 100, 500, 1000],
    cv = 5,
    scoring = 'neg_mean_squared_error'
)
best_estimator.fit(X_train_val, y_train_val)
predicted = best_estimator.predict(X_test)
print(mean_squared_error(predicted, y_test))<jupyter_output>0.010791103067324314<jupyter_text>**Neural network approach**<jupyter_code>model = Sequential()
model.add(Dense(30, input_dim=train.shape[1], activation='relu'))
model.add(Dense(20, activation='relu'))
model.add(Dense(15, activation='relu'))
#model.add(Dense(10, kernel_initializer='normal', activation='relu'))
model.add(Dense(4, kernel_initializer='normal', activation='relu'))
model.add(Dense(1, kernel_initializer='normal', activation='linear'))
model.compile(
    loss='mean_squared_error',
    optimizer='adam'
)
plot_model(model, show_shapes=True)
np.random.seed(123)
history = model.fit(train, y, epochs=80, batch_size=32, verbose=1, validation_split=0.25)
plt.figure(figsize=(10, 10))
plt.plot(history.history['loss'][5:])
plt.plot(history.history['val_loss'][5:])
plt.legend(['train', 'test'])
history.history['val_loss'][-1]
#best_estimator = model<jupyter_output><empty_output><jupyter_text>**Submission**<jupyter_code>best_estimator.fit(train, y)
final_predictions = np.exp(best_estimator.predict(test))
sample_submission = pd.read_csv(sample_submission_path, index_col='Id')
submission = pd.DataFrame({
    'Id': sample_submission.index,
    'SalePrice': final_predictions
})
submission.to_csv(submission_path, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13057427013153675, 'fraction_numerical': 0.02823227462303497, 'mean_word_length': 6.54404646660213, 'pattern_counts': {'":': 0, '<': 95, '<?xml version=': 0, '>': 98, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '142265', 'license': 'MIT', 'n_tokens_mistral': 5860, 'n_tokens_neox': 5435, 'n_words': 988, 'path': 'House_Prices.ipynb', 'repo_name': 'PashaIanko/Kaggle.HousePrices', 'size': '15585'}