<jupyter_start><jupyter_text>Preprocess train dataset<jupyter_code>import pandas as pd
from pathlib import Path
import numpy as np

#Math tools
from scipy import stats
from scipy.stats import skew,norm  # for some statistics
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
import scipy.stats as stats
def ingest_raw_data():
    path = Path()
    i = 1
    d = {}
    for file in path.glob('raw_data/*.csv'):
        d[str(i)] = pd.read_csv(file)
        print(f'Loaded df{str(i)}: {file}')
        i += 1
    return d
d = ingest_raw_data()
test = d['2'][:]
df = d['3'][:]
df_ID = df['Id']
df.drop(['Id'], axis=1, inplace=True)<jupyter_output>C:\Users\fmhiga\AppData\Roaming\Python\Python37\site-packages\pandas\core\frame.py:3997: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  errors=errors,<jupyter_text>Missing Data<jupyter_code>#missing data
total = df.isnull().sum().sort_values(ascending=False)
percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.head(20)
#missing data
total = test.isnull().sum().sort_values(ascending=False)
percent = (test.isnull().sum()/test.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.head(20)
# PoolQC: Pool quality
df['PoolQC'].value_counts()
# MiscFeature: Miscellaneous feature not covered in other categories
df['MiscFeature'].value_counts()
# Alley: Type of alley access
df['Alley'].value_counts()
# Fence: Fence quality
df['Fence'].value_counts()
# FireplaceQu: Fireplace quality
df['FireplaceQu'].value_counts()
# LotFrontage: Linear feet of street connected to property
# Neighborhood: Physical locations within Ames city limits
df['LotFrontage'].value_counts()
# GarageCond: Garage condition
df['GarageCond'].value_counts()
# GarageCond: Garage condition
df['GarageQual'].value_counts()
# GarageYrBlt: Year garage was built
df['GarageYrBlt'].value_counts()
# GarageFinish: Interior finish of the garage
df['GarageFinish'].value_counts()
# GarageType: Garage location
df['GarageType'].value_counts()
# BsmtCond: General condition of the basement
df['BsmtCond'].value_counts()
# BsmtQual: Height of the basement
df['BsmtQual'].value_counts()
# BsmtExposure: Walkout or garden level basement walls
df['BsmtExposure'].value_counts()
# BsmtFinType1: Quality of basement finished area
df['BsmtFinType1'].value_counts()
# BsmtFinType2: Quality of second finished area (if present)
df['BsmtFinType2'].value_counts()
# MasVnrArea: Masonry veneer area in square feet
df['MasVnrArea'].value_counts()
# Electrical: Electrical system
df['Electrical'].value_counts()
# Electrical: Electrical system
df['MSZoning'].value_counts()
# BsmtHalfBath: Basement half bathrooms
df['BsmtHalfBath'].value_counts()
print("Total No. of missing value {} before Imputation".format(sum(df.isnull().sum())))

class missing_values():
    
    for col_cat in ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageCond', 'GarageQual', 'GarageYrBlt',
               'GarageFinish', 'GarageType', 'BsmtCond', 'BsmtQual', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
               'MasVnrType', 'MSSubClass']:
        df[col_cat] = df[col_cat].fillna('None')
    
    # LotFrontage : Since the area of each street connected to the house property most likely have a similar area to 
    # other houses in its neighborhood , we can fill in missing values by the median LotFrontage of the neighborhood.
    #Group by neighborhood and fill in missing value by the median LotFrontage of all the neighborhood
    df["LotFrontage"] = df.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
    df['Electrical'] = df['Electrical'].fillna(df['Electrical'].mode()[0])
    # MSZoning (The general zoning classification) : 'RL' is by far the most common value. So we can fill in 
    # missing values with 'RL'
    df['MSZoning'] = df['MSZoning'].fillna(df['MSZoning'].mode()[0])
    # Electrical : It has one NA value. Since this feature has mostly 'SBrkr', we can set that for the missing value.
    df['Electrical'] = df['Electrical'].fillna(df['Electrical'].mode()[0])
    # Functional : data description says NA means typical
    df["Functional"] = df["Functional"].fillna("Typ")
    # KitchenQual: Only one NA value, and same as Electrical, we set 'TA' (which is the most frequent) for the missing 
    # value in KitchenQual.
    df['KitchenQual'] = df['KitchenQual'].fillna(df['KitchenQual'].mode()[0])
    # Exterior1st and Exterior2nd : Again Both Exterior 1 & 2 have only one missing value. We will just substitute in 
    # the most common string
    df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode()[0])
    df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode()[0])
    # SaleType : Fill in again with most frequent which is "WD"
    df['SaleType'] = df['SaleType'].fillna(df['SaleType'].mode()[0])

    for col_num in ['MasVnrArea', 'BsmtHalfBath']:
        df[col_num] = df[col_num].fillna(0)
    
missing_values()

print("Total No. of missing value {} after Imputation".format(sum(df.isnull().sum())))
df['KitchenQual'].isna().sum()<jupyter_output><empty_output><jupyter_text>Change number to categorial features<jupyter_code># converting some numeric features to string

#MSSubClass: The building class
df['MSSubClass'] = df['MSSubClass'].apply(str)
# YrSold: Year Sold
df['YrSold'] = df['YrSold'].astype(str)
#MoSold: Month Sold
df['MoSold'] = df['MoSold'].astype(str)
# GarageYrBlt: Year garage was built
df['GarageYrBlt'] = df['GarageYrBlt'].astype(str)<jupyter_output>C:\ProgramData\Anaconda3\lib\site-packages\ipykernel_launcher.py:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  after removing the cwd from sys.path.
C:\ProgramData\Anaconda3\lib\site-packages\ipykernel_launcher.py:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  
C:\ProgramData\Anaconda3\lib\site-packages\ipykernel_launcher.py:8: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the docum[...]<jupyter_text>Feature Engineering<jupyter_code># YrSold: Year Sold Original construction date + YearRemodAdd: Remodel date
df['YearsSinceRemodel'] = df['YrSold'].astype(int) + df['YearRemodAdd'].astype(int)
# OverallQual: Overall material and finish quality + OverallCond: Overall condition rating
df['Total_Home_Quality'] = df['OverallQual'] + df['OverallCond']
# TotalBsmtSF: Total square feet of basement area + 1stFlrSF: First Floor square feet + 2ndFlrSF: Second floor square feet
df['TotalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
# YearBuilt: Original construction date + YearRemodAdd: Remodel date
df['YrBltAndRemod'] = df['YearBuilt'] + df['YearRemodAdd']
# BsmtFinSF1: Type 1 finished square feet + BsmtFinSF2: Type 2 finished square feet + 
# 1stFlrSF: First Floor square feet + 2ndFlrSF: Second floor square feet
df['Total_sqr_footage'] = (df['BsmtFinSF1'] + df['BsmtFinSF2'] + df['1stFlrSF'] + df['2ndFlrSF'])
# FullBath: Full bathrooms above grade + HalfBath: Half baths above grade + 
# BsmtFullBath: Basement full bathrooms + BsmtHalfBath: Basement half bathrooms
df['Total_Bathrooms'] = (df['FullBath'] + (0.5 * df['HalfBath']) + df['BsmtFullBath'] + (0.5 * df['BsmtHalfBath']))
# OpenPorchSF: Open porch area in square feet + 3SsnPorch: Three season porch area in square feet +
# EnclosedPorch: Enclosed porch area in square feet + ScreenPorch: Screen porch area in square feet + 
# WoodDeckSF: Wood deck area in square feet
df['Total_porch_sf'] = (df['OpenPorchSF'] + df['3SsnPorch'] + df['EnclosedPorch'] + df['ScreenPorch'] + df['WoodDeckSF'])
# simplified features
df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
df['hasgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
df['hasbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
df['hasfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1460 entries, 0 to 1459
Data columns (total 92 columns):
 #   Column              Non-Null Count  Dtype  
---  ------              --------------  -----  
 0   MSSubClass          1460 non-null   object 
 1   MSZoning            1460 non-null   object 
 2   LotFrontage         1460 non-null   float64
 3   LotArea             1460 non-null   int64  
 4   Street              1460 non-null   object 
 5   Alley               1460 non-null   object 
 6   LotShape            1460 non-null   object 
 7   LandContour         1460 non-null   object 
 8   Utilities           1460 non-null   object 
 9   LotConfig           1460 non-null   object 
 10  LandSlope           1460 non-null   object 
 11  Neighborhood        1460 non-null   object 
 12  Condition1          1460 non-null   object 
 13  Condition2          1460 non-null   object 
 14  BldgType            1460 non-null   object 
 15  HouseStyle          1460 non-null   object 
 16  Overal[...]<jupyter_text>Skewed features In probability theory and statistics, skewness is a measure of the asymmetry of the probability distribution of a real-valued random variable about its mean. The skewness value can be positive or negative, or undefined.For a unimodal distribution, negative skew commonly indicates that the tail is on the left side of the distribution, and positive skew indicates that the tail is on the right. In cases where one tail is long but the other tail is fat, skewness does not obey a simple rule. For example, a zero value means that the tails on both sides of the mean balance out overall; this is the case for a symmetric distribution, but can also be true for an asymmetric distribution where one tail is long and thin, and the other is short but fat.<jupyter_code># Filter the skewed features
numeric = df.select_dtypes(include='number').columns
skew_features = df[numeric].apply(lambda x: skew(x)).sort_values(ascending=False)

high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index

print("There are {} numerical features with Skew > 0.5 :".format(high_skew.shape[0]))
skewness = pd.DataFrame({'Skew' :high_skew})
skew_features.head(10)<jupyter_output>There are 30 numerical features with Skew > 0.5 :<jupyter_text>Box Cox Transformation of (highly) skewed features Performing Box Cox transformations is a powerful and elegant way of normalizing skewed data and can lead to significant improvements in machine learning performance.<jupyter_code># Normalize skewed features using boxcox
for i in skew_index:
    df[i] = boxcox1p(df[i], boxcox_normmax(df[i] + 1))<jupyter_output>C:\ProgramData\Anaconda3\lib\site-packages\scipy\stats\stats.py:3508: PearsonRConstantInputWarning: An input array is constant; the correlation coefficent is not defined.
  warnings.warn(PearsonRConstantInputWarning())
C:\ProgramData\Anaconda3\lib\site-packages\scipy\stats\stats.py:3538: PearsonRNearConstantInputWarning: An input array is nearly constant; the computed correlation coefficent may be inaccurate.
  warnings.warn(PearsonRNearConstantInputWarning())<jupyter_text>Getting dummy categorical features<jupyter_code># Get dummies
df = pd.get_dummies(df, prefix_sep='_', drop_first=True)

# prefix_sep='_',
df.shape
df.info()
def remove_overfit_features(df,weight):
    overfit = []
    for i in df.columns:
        counts = df[i].value_counts()
        zeros = counts.iloc[0]
        if zeros / len(df) * 100 > weight:
            overfit.append(i)
    overfit = list(overfit)
    return overfit

overfitted_features = remove_overfit_features(df,99)
overfitted_features
for c in df.columns:
    print(c)
df.drop(overfitted_features, inplace=True, axis=1)
df.shape
df.to_csv('preprossed_data/preprossed_data.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11979783621574666, 'fraction_numerical': 0.017373450209271106, 'mean_word_length': 4.686573866187697, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 34, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '348836', 'license': 'MIT', 'n_tokens_mistral': 4379, 'n_tokens_neox': 4117, 'n_words': 1182, 'path': '2. Preprocess train dataset .ipynb', 'repo_name': 'fmhiga/House-Prices', 'size': '12663'}