<jupyter_start><jupyter_text>Importing Libraries<jupyter_code>import os
from pathlib import Path

# data manipulation
import pandas as pd
import numpy as np
np.set_printoptions(suppress=True)

# plotting
import seaborn as sn
import matplotlib.pyplot as plt
%matplotlib inline

import yaml

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelBinarizer<jupyter_output>/Users/Truoc_Pham/Documents/kaggle-house-prices-dvc/venv37/lib/python3.7/site-packages/pandas/compat/__init__.py:120: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.
  warnings.warn(msg)<jupyter_text>Load categorized dataset<jupyter_code>train_df = pd.read_csv('../data/interim/train_categorized.csv')
test_df = pd.read_csv('../data/interim/test_categorized.csv')
train_df.shape, test_df.shape
train_df.head()<jupyter_output><empty_output><jupyter_text>Replace NaN  Numerical<jupyter_code>def is_missing(df, columns):
    """Helper function to check missing values on dataset"""
    for column in columns:
        if df[column].isnull().values.any():
            return True
        else:
            return False
def load_params(filepath='params.yaml') -> dict:
    """Helper function to load params.yaml

    Args:
        filepath (str): filename or full filepath to yaml file with parameters

    Returns:
        dict: dictionary of parameters
    """

    assert (os.path.isfile(filepath)), FileNotFoundError

    # read params.yaml
    with open(filepath, 'r') as file:
        params = yaml.safe_load(file)

    return params
params = load_params('../params.yaml')
params['ignore_cols']
params['imputation']['method']
# concatenate df
df = pd.concat([train_df, test_df], ignore_index=True)

# fill NaNs with the default strategy is mean
num_cols = df.select_dtypes(include=[np.number]).columns.difference(params['ignore_cols'])
imputer = SimpleImputer(missing_values=np.NaN, strategy=params['imputation']['method'])
for col in num_cols:
    # fit imputing with numerical column
    imputer = imputer.fit(df[[col]])

    # assign imputed value for numerical column
    df[col] = imputer.transform(df[[col]]).ravel()

# make sure no missing values
assert (not is_missing(df, df.columns)), AssertionError

# return datasets to train and test
n_train = train_df.shape[0]
train_df = df[:n_train]
test_df = df[n_train:]
train_df.shape, test_df.shape
train_df.head()
train_df = replace_num_missing(train_df)
test_df = replace_num_missing(test_df)
train_total_missing = train_df.isnull().sum().sort_values(ascending=False)
train_ratio_missing = (train_df.isnull().sum()/train_df.isnull().count()).sort_values(ascending=False)
missing_train_data = pd.concat([train_total_missing, train_ratio_missing], axis=1, keys=['Total', 'Ratio'])
missing_train_data['Type'] = train_df[missing_train_data.index].dtypes

missing_train_data = missing_train_data[(missing_train_data['Total'] > 0)]

# view missing data stats
print('=== Missing Train Data Stats ===')
missing_train_data

print('Numerical Missing Values:')
print('=========================')
[print(col_missing,  '\t', missing_train_data['Total'][col_missing], 'NaNs') \
 for col_missing in missing_train_data[(missing_train_data['Total'] > 0) & \
                                 (missing_train_data['Type'] != 'object')].index.values]
print('=========================')
# Statistics missing values of train after impute
# Count unique missing value of each column
count = 0
for col in train_df.columns:
    if train_df[col].isnull().values.any():
        print(col)
        print(train_df[col].isnull().sum())
        count += 1
        
print('==========================')
print('Missing counter %d' % count)<jupyter_output>==========================
Missing counter 0
{'fraction_non_alphanumeric': 0.14011761697775504, 'fraction_numerical': 0.0035796471490667347, 'mean_word_length': 5.160629921259843, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '315778', 'license': 'MIT', 'n_tokens_mistral': 1246, 'n_tokens_neox': 1203, 'n_words': 286, 'path': 'notebooks/03_impute_nan_playground.ipynb', 'repo_name': 'truocpham-agilityio/kaggle-house-prices-dvc', 'size': '3911'}