<jupyter_start><jupyter_text>Preprocessing> Contains all the steps necessary to do some basic pre-processing for the Kaggle house prices data.<jupyter_code>#default_exp preprocessing
#hide
%load_ext autoreload
%autoreload 2
%matplotlib inline
#export
from zipfile import ZipFile
from pathlib import Path
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
from typing import List
from fastai2.tabular.all import *

from kaggle_house_prices.utils import *
mpl.style.use("bmh")<jupyter_output><empty_output><jupyter_text>Downloading & extracting zip files<jupyter_code>#export
dir_raw_data = Path("../data/raw")
# competition = "house-prices-advanced-regression-techniques"
compressed_data_file = dir_raw_data/f"{competition}.zip"
#hide
compressed_data_file
#hide
# !kaggle competitions download -c {competition} -p {dir_raw_data}
#hide
# extract_zip_files(compressed_data_file)
#export
train_path = dir_raw_data/"train.csv"
test_path = dir_raw_data/"test.csv"
#hide
train_path, test_path<jupyter_output><empty_output><jupyter_text>Loading into data frames and visualizing<jupyter_code>#hide
df = pd.read_csv(train_path)
df_test = pd.read_csv(test_path)
#hide
display_all(df.head().T)
#hide
display_all(df_test.head().T)
#export
dep_var = "SalePrice"
#export
def show_dep_var(df:pd.DataFrame, dep_var:str, bins:int=50):
    "Shows the `dep_var` distribution, linear and logarithmic, side by side."
    bins = 50

    fig, axs = plt.subplots(ncols=2, figsize=(14,4))
    ax = axs[0]
    ax.hist(df[dep_var].values, bins=bins)
    ax.set_title(f"{dep_var}")
    ax = axs[1]
    ax.hist(np.log(df[dep_var].values), bins=bins)
    ax.set_title(f"log({dep_var})")
    plt.show()
show_dep_var(df, dep_var, bins=50)
#hide
df[dep_var] = df[dep_var].apply(np.log)<jupyter_output><empty_output><jupyter_text>Cleaning the data Imputing NaN values for continuous columns and flagging of NaNs with new `_na` columns.<jupyter_code>#export
cont_names = ['LotArea', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF2', 'BsmtUnfSF',
              'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GrLivArea', 'GarageYrBlt', 'GarageArea',
              'WoodDeckSF', 'OpenPorchSF', 'LotFrontage', 'BsmtFinSF1', 'LowQualFinSF',]
display_all(df.loc[:, cont_names].describe().T)
#export
def show_na_share(df:pd.DataFrame, cols:List[str]):
    "Prints the share of NaN values by column"
    for col in cols: 
        if df[col].hasnans: 
            print(f"{col}: {df[col].isna().sum()/len(df)*100:.2f} %")
show_na_share(df, cont_names)
#export
cat_names = ['Id', 'MSSubClass', 'MSZoning', 'Street',
       'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig',
       'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType',
       'HouseStyle', 'OverallQual', 'OverallCond', 
       'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType',
       'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual',
       'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 
       'BsmtFinType2', 'Heating',
       'HeatingQC', 'CentralAir', 'Electrical', 
       'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
       'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual',
       'TotRmsAbvGrd', 'Functional', 'Fireplaces', 'FireplaceQu', 'GarageType',
       'GarageFinish', 'GarageCars', 'GarageQual',
       'GarageCond', 'PavedDrive',
       'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'PoolQC',
       'Fence', 'MiscFeature', 'MiscVal', 'MoSold', 'SaleType',
       'SaleCondition']
show_na_share(df, cat_names)<jupyter_output>Alley: 93.77 %
MasVnrType: 0.55 %
BsmtQual: 2.53 %
BsmtCond: 2.53 %
BsmtExposure: 2.60 %
BsmtFinType1: 2.53 %
BsmtFinType2: 2.60 %
Electrical: 0.07 %
FireplaceQu: 47.26 %
GarageType: 5.55 %
GarageFinish: 5.55 %
GarageQual: 5.55 %
GarageCond: 5.55 %
PoolQC: 99.52 %
Fence: 80.75 %
MiscFeature: 96.30 %<jupyter_text>Setting categories<jupyter_code>for col in cat_names:
    df[col] = pd.Categorical(df[col], ordered=True)<jupyter_output><empty_output><jupyter_text>Sub-sampling To accelerate the turnaround time during model development lets do some sub-sampling of `n` samples, without replacement.<jupyter_code>#hide
small_sample = True
n = 100
#export
def do_sample(df:pd.DataFrame, small_sample:bool, n:int=100) -> pd.DataFrame:
    "Generates samples from `df`."
    if small_sample:
        _df = df.sample(n=n)
    else:
        _df = df.copy(deep=True)
    return _df
df_s = do_sample(df, small_sample=small_sample, n=n)
len(df_s), len(df)
#hide
assert len(df_s) == n
#hide
assert len(set(df.columns).symmetric_difference(df_s.columns)) == 0<jupyter_output><empty_output><jupyter_text>Splitting into train and validation set Using `get_split`, the data is split using the underlying order adding a `valid` columns indicating the set affiliation. If the dataframe is ordered, e.g. increasingly along a time column, then `valid_pct` determines the number of the last rows to use in the validation set. All values before are then training samples.<jupyter_code>#export
val_col = "is_valid"

def get_split(df:pd.DataFrame, valid_pct:float=.2, val_col="is_valid") -> pd.DataFrame:
    "Splitting the data frame into train and validation set, creating a new `val_col` column"
    valid_idx = np.arange(int((1-valid_pct)*len(df)), len(df))
    train_idx = np.setdiff1d(np.arange(len(df)), valid_idx)
    df[val_col] = False
    df.iloc[valid_idx, df.columns.get_loc(val_col)] = True
    return df
df_s = get_split(df_s, valid_pct=.2)
#hide
assert len(df_s.columns) == len(df.columns)+1
df_s[val_col].value_counts()<jupyter_output><empty_output><jupyter_text>Dealing with unexpected missing values (continuous)<jupyter_code>#export
def deal_with_continuous_nans(df_ref:pd.DataFrame, df_new, cont_names:list, cont_fill_vals:dict):
    "Fills nans in columns of `df_new` if they are unexpected based on `df_ref`"
    for col in cont_names:
        reference_set_has_nan = pd.isnull(df_ref[col]).any()
        new_set_has_nan = pd.isnull(df_new[col]).any()
        
        if new_set_has_nan and not reference_set_has_nan:
            df_new[col].fillna(cont_fill_vals[col], inplace=True)
    return df_new
%%time
cont_fill_vals = {col: np.nanmedian(df_s.loc[df_s[val_col]==False, col]) for col in cont_names}
cont_fill_vals
ix_train = df_s[val_col]==False
df_s.loc[~ix_train,:] = deal_with_continuous_nans(df_s.loc[ix_train,:], df_s.loc[~ix_train,:], 
                                                  cont_names, cont_fill_vals)
splits = ColSplitter(col=val_col)(df_s)
to = TabularPandas(df_s, procs=[Categorify, FillMissing, Normalize], cat_names=cat_names, cont_names=cont_names,
                   y_names=dep_var, splits=splits)<jupyter_output><empty_output><jupyter_text>Inspecting `TabularPandas` mappinds Going over categorical mappings where `Categorify` did not make one-hot encodings sanity checking train and validation set encodings<jupyter_code>#export
def inspect_category_mappings(cat_names:List[str], df:pd.DataFrame, splits:ColSplitter, 
                              to:TabularPandas, n_max:int=None):
    "Shows unique original and transformed discrete values side by side for sanity checks"
    for i, col in enumerate(cat_names):
        print(f"\nColumn: {col}")
        _df = (pd.DataFrame({"original-train": df.iloc[splits[0]][col], "categorified-train": to.train.xs[col]})
               .drop_duplicates()
               .sort_values("categorified-train"))
        display_all(_df.head())
        _df = (pd.DataFrame({"original-valid": df.iloc[splits[1]][col], "categorified-valid": to.valid.xs[col]})
               .drop_duplicates()
               .sort_values("categorified-valid"))
        display_all(_df.head())

        if n_max is not None and i+1 == n_max:
            break
inspect_category_mappings(cat_names, df_s, splits, to, n_max=1)<jupyter_output>Column: Id<jupyter_text>Exporting `TabularPandas` instanceNecessary step for the subsequent modelling step.<jupyter_code>#export
model_data_path = "model_data.pckl"
with open(model_data_path, "wb") as f:
    pickle.dump(to, f)<jupyter_output><empty_output><jupyter_text>Exporting to csvThis can be useful as an intermediate step if you want to look at the data using another visualization tool. Using a visualization tool can be useful to inspect any unwanted artfiacts or skewness of the cleaned data.<jupyter_code>#export
clean_data_path = dir_raw_data.parent/"train_clean.csv"
clean_test_data_path = dir_raw_data.parent/"test_clean.csv"
clean_data_path_feather = dir_raw_data.parent/"train_clean.feather"
clean_test_data_path_feather = dir_raw_data.parent/"test_clean.feather"
clean_data_path, clean_test_data_path
df_s.to_csv(clean_data_path, index=False, encoding="utf8")
df_test.to_csv(clean_test_data_path, index=False, encoding="utf8")
df_s.reset_index().to_feather(clean_data_path_feather)
df_test.reset_index().to_feather(clean_test_data_path_feather)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12625698324022347, 'fraction_numerical': 0.011396648044692738, 'mean_word_length': 5.076714188730482, 'pattern_counts': {'":': 4, '<': 43, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '534417', 'license': 'Apache-2.0', 'n_tokens_mistral': 3405, 'n_tokens_neox': 3253, 'n_words': 678, 'path': 'notebooks/01_preprocessing.ipynb', 'repo_name': 'eschmidt42/kaggle_house_prices', 'size': '8950'}