<jupyter_start><jupyter_text>Method 2,3,4 -> Using scikit learn module (all strategies at once)<jupyter_code>import numpy as np
import pandas as pd
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
train=pd.read_csv(r"C:\Users\lalit\Documents\JUPYTER NOTEBOOK\house-prices-advanced-regression-techniques\train.csv")
test=pd.read_csv(r"C:\Users\lalit\Documents\JUPYTER NOTEBOOK\house-prices-advanced-regression-techniques\test.csv")
print("shape of train df = ",train.shape)
print("shape of test df = ",test.shape)
# dataset link -> https://drive.google.com/drive/folders/1gdfNSb3ptz0WvShshUeiM-kqfMYzQ5tm?usp=sharing
# convert data into X_train y_train
X_train=train.drop(columns="SalePrice",axis=1)
y_train=train["SalePrice"]
X_test=test.copy()
print("shape of X_train df = ",X_train.shape)
print("shape of y_train df = ",y_train.shape)
print("shape of X_test df = ",X_test.shape)<jupyter_output>shape of X_train df =  (1460, 80)
shape of y_train df =  (1460,)
shape of X_test df =  (1459, 80)<jupyter_text>Missing value imputation<jupyter_code>isnull_sum=X_train.isnull().sum()
isnull_sum
# names of columns having missing values -> numerical 
num_vars=X_train.select_dtypes(include=["int64","float64"]).columns
num_vars_miss=[var for var in num_vars if isnull_sum[var]>0]
num_vars_miss
# names of columns having missing values -> numerical
cat_vars=X_train.select_dtypes(include=["object"]).columns
cat_vars_miss=[var for var in cat_vars if isnull_sum[var]>0]
cat_vars_miss
num_var_mean=["LotFrontage"] # mean for LotFrontage
num_var_median= ["MasVnrArea",'GarageYrBlt']  # median for both
cat_var_mode = ['Alley',
 'MasVnrType',
 'BsmtQual',
 'BsmtCond',
 'BsmtExposure',
 'BsmtFinType1',
 'BsmtFinType2',
 'Electrical',
 'FireplaceQu'] # mode value 
cat_var_const= ['GarageType',
 'GarageFinish',
 'GarageQual',
 'GarageCond',
 'PoolQC',
 'Fence',
 'MiscFeature']  # const value
# create pipeline to fix the sequence 
num_var_mean_imputer = Pipeline(steps=[("imputer",SimpleImputer(strategy="mean"))])
num_var_median_imputer = Pipeline(steps=[("imputer",SimpleImputer(strategy="median"))])
cat_var_mode_imputer = Pipeline(steps=[("imputer",SimpleImputer(strategy="most_frequent"))])
cat_var_const_imputer =Pipeline(steps=[("imputer",SimpleImputer(strategy="constant",fill_value="KHALI COLUMN"))])

# Transform the pipeline to columns
# transformers ("name", startegy , columns)
prep = ColumnTransformer(transformers=[("mean_imputer",num_var_mean_imputer,num_var_mean),
                               ("median_imputer",num_var_median_imputer,num_var_median),
                               ("mode_imputer",cat_var_mode_imputer,cat_var_mode),
                               ("const_imputer",cat_var_const_imputer,cat_var_const)])
# fit the transforms  
prep.fit(X_train)
# detailed info
prep.transform
# getting the values that r filled 
prep.named_transformers_["mean_imputer"].named_steps["imputer"].statistics_
train["LotFrontage"].mean()
prep.named_transformers_["mode_imputer"].named_steps["imputer"].statistics_
X_train_clean =prep.transform(X_train)
X_test_clean =prep.transform(X_test)
X_train_clean
prep.transformers_

# drop -> rows or columns skipped as they dont have missing values
X_train_clean_DF =pd.DataFrame(X_train_clean,columns=num_var_mean+num_var_median+cat_var_mode+cat_var_const)
X_train_clean_DF
X_train_clean_DF.isnull().sum().sum()
# check values filled are correct or not
train["Alley"].value_counts()
X_train_clean_DF["Alley"].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12548910005589714, 'fraction_numerical': 0.008943543879262158, 'mean_word_length': 6.918141592920354, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '487066', 'license': 'MIT', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1284, 'n_words': 226, 'path': 'Data Filling and Cleaning Techniques/DATA CLEANING-6 Scikit Learn Module.ipynb', 'repo_name': 'kaustavr19/Hands-on-ML-Basic-to-Advance-', 'size': '3578'}