<jupyter_start><jupyter_text>--- Data Prep  Dataset Cleaning<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

%matplotlib inline

from time import time
from src.features import build_features as bf
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import fbeta_score, accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer, MissingIndicator
from sklearn.pipeline import FeatureUnion, make_pipeline

sns.set()<jupyter_output><empty_output><jupyter_text>--- Data Preprocessing<jupyter_code>features_raw = pd.read_csv('../data/interim/features_raw.csv', index_col='Id')
target = pd.read_csv('../data/interim/target_raw.csv', index_col='Id', squeeze=True)
features_raw.head()
features_raw.shape
df_zscore = pd.read_csv('../data/interim/df_zscore.csv', index_col='Id')
outlier_idx = df_zscore[(df_zscore >= 5).any(1)].index
outlier_idx<jupyter_output><empty_output><jupyter_text>Handle Outliers<jupyter_code>df = features_raw.drop(index=outlier_idx)
target = target.drop(index=outlier_idx)
index = df.index

# Uncomment this line to save the DataFrames
# target.to_csv('../data/interim/target_no_outliers.csv')
df.shape<jupyter_output><empty_output><jupyter_text>Assess Missing Data  Assess Missing Data in Each Column<jupyter_code>df.isna().sum().sort_values(ascending=False).head(10)<jupyter_output><empty_output><jupyter_text>Assess Missing Data in Each Column<jupyter_code>nan_count = df.isna().sum()
nan_count = nan_count[nan_count > 0]
nan_cols = df[nan_count.index].columns
(nan_count / df.shape[0]).sort_values(ascending=False)
# Investigate patterns in the amount of missing data in each column.

# plt.rcParams.update({'figure.dpi':100})

plt.figure(figsize=(9, 8))
ax = sns.histplot(nan_count, kde=False)
ax.set_title('Histogram of Missing Data by Column')
ax.set(xlabel='Total Missing or Unknown', ylabel='Total Occurrences')
plt.show()

# Uncomment this line to save the figure.
# plt.savefig('../reports/figures/MissingDatabyCol_Histogram.svg')
nan_rows = df.isna().sum(axis=1)
plt.figure(figsize=(9, 8))
ax = sns.histplot(nan_rows, kde=False)
ax.set_title('Histogram of Missing Data by Row')
ax.set(xlabel='Total Missing or Unknown', ylabel='Total Occurrences')
plt.show()

# Uncomment this line to save the figure.
# plt.savefig('../reports/figures/MissingDatabyRow_Histogram.svg')<jupyter_output><empty_output><jupyter_text>Assessment SummaryThere is a fair amount of missing data in this dataset. Four features in particular (PoolQC, MiscFeature, Alley, Fence) contain >50% missing or unknown values. For PoolQC, we may be able to imply whether or not the home has a pool performing some feature engineering on the PoolArea feature. In addition, several features such as 'GarageArea' indicate the total square footage of a garage (if any) but our dataset does not seem to indicate whether a not a particular home has a garage or not. We'll create features for these. Let's investigate these in turn.<jupyter_code>df['PoolArea'].value_counts()<jupyter_output><empty_output><jupyter_text>Most homes don't have pools. Let's use the value of 'PoolArea' to assign 'NA' to 'PoolQC' if the area == 0<jupyter_code>df['PoolQC'].value_counts()
df.loc[df['PoolArea'] == 0, 'PoolQC'] = 'NA' # If no 'PoolArea', (0), then there is no pool 
df['PoolQC'].value_counts()<jupyter_output><empty_output><jupyter_text>Similarly, let's apply the same logic to other features where necessary:<jupyter_code>df['Fence'].value_counts()
df['Alley'].value_counts()
df['BsmtCond'].value_counts()
df['FireplaceQu'].value_counts()
df.loc[df['Fence'].isna(), 'Fence'] = 'NA'
df.loc[df['Alley'].isna(), 'Alley'] = 'NA'
df.loc[df['BsmtQual'].isna(), 'BsmtQual'] = 'NA'
df.loc[df['BsmtCond'].isna(), 'Bsmtcond'] = 'NA'
df.loc[df['BsmtExposure'].isna(), 'BsmtExposure'] = 'NA'
df.loc[df['BsmtFinType1'].isna(), 'BsmtFinType1'] = 'NA'
df.loc[df['BsmtFinType2'].isna(), 'BsmtFinType2'] = 'NA'
df.loc[df['FireplaceQu'].isna(), 'FireplaceQu'] = 'NA'

df['HasFence'] = df['Fence']
df['HasFence'].apply(lambda x: 1 if x != 'NA' else 0)

df['HasAlley'] = df['Alley']
df['HasAlley'].apply(lambda x: 1 if x != 'NA' else 0)

df['HasPool'] = df['PoolArea']
df['HasPool'].apply(lambda x: 1 if x > 0 else 0)

df['HasGarage'] = df['GarageArea']
df['HasGarage'].apply(lambda x: 1 if x > 0 else 0)

df['HasBasement'] = df['TotalBsmtSF']
df['HasBasement'].apply(lambda x: 1 if x > 0 else 0)

df['HasFireplace'] = df['FireplaceQu']
df['HasFireplace'].apply(lambda x: 1 if x != 'NA' else 0)
categorical_cols = df.select_dtypes(include=object).columns
numerical_cols = df.select_dtypes(include=np.number).columns
# Perform One-Hot Encoding on our Categorical Data

features_enc = df.copy()
onehot_df = features_enc.drop(columns=categorical_cols)
features_onehot_enc = pd.get_dummies(features_enc[categorical_cols], dummy_na=True)
onehot_df[features_onehot_enc.columns] = features_onehot_enc

# Uncomment this line to export DataFrame
# onehot_df.to_csv('../data/interim/onehot_df.csv')

# Print the number of features after one-hot encoding
encoded = list(onehot_df.columns)
print(f'{len(encoded)} total features after one-hot encoding.')

# Uncomment the following line to see the encoded feature names
# print(encoded)
onehot_df.isna().sum().sort_values().tail(3)
nan_labels = ['MasVnrArea_nan', 'GarageYrBlt_nan', 'LotFrontage_nan']
encoded.extend(nan_labels)
imp = IterativeImputer(missing_values=np.nan, random_state=5, max_iter=20, add_indicator=True)
imputed_arr = imp.fit_transform(onehot_df)
features_imputed = pd.DataFrame(imputed_arr, columns=encoded)
features_imputed.index = onehot_df.index

# Uncomment this line to export DataFrame
# features_imputed.to_csv('../data/interim/features_imputed.csv')<jupyter_output><empty_output><jupyter_text>Feature Transformation Transforming Skewed Continuous FeaturesA dataset may sometimes contain at least one feature whose values tend to lie near a single number, but will also have a non-trivial number of vastly larger or smaller values than that single number. Algorithms can be sensitive to such distributions of values and can underperform if the range is not properly normalized. We'll need to check the following continuous data features for 'skew'.- LotFrontage- LotArea- MasVnrArea- BsmtFinSF1- BsmtFinSF2- TotalBsmtSF- 1stFlrSF- 2ndFlrSF- LowQualFinSF- GrLivArea- GarageArea- WoodDeckSF- OpenPorchSF- EnclosedPorch- 3SsnPorch- ScreenPorch- PoolArea- MiscVal<jupyter_code>continuous_cols = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'TotalBsmtSF', '1stFlrSF', 
        '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF',
        'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']

skew = features_imputed[continuous_cols].skew().sort_index()
skew
# Show columns with skew greater than 2 * Standard Error
skew > 2 * features_imputed[continuous_cols].sem().sort_index()
skewed = ['ScreenPorch', 'PoolArea', 'LotFrontage', '3SsnPorch', 'LowQualFinSF']

fig = plt.figure(figsize = (16,10));

# Skewed feature plotting
for i, feature in enumerate(skewed):
    ax = fig.add_subplot(2, 3, i+1)
    sns.histplot(features_imputed[feature], bins=20, color='#00A0A0')
    ax.set_title("'%s' Feature Distribution"%(feature), fontsize = 14)
    ax.set_xlabel("Value")
    ax.set_ylabel("Number of Records")
    ax.set_ylim((0, 1600))
    ax.set_yticks([0, 400, 800, 1200, 1600])
    ax.set_yticklabels([0, 400, 800, 1200, ">1600"])

# Plot aesthetics
fig.suptitle("Skewed Distributions of Continuous Data Features", \
        fontsize = 16, y = 1.03)

fig.tight_layout()

# Uncomment this line to save the figure.
# plt.savefig('../reports/figures/Skewed_Distributions.svg')
features_log_xformed = pd.DataFrame(data = features_imputed)

# Since the logarithm of 0 is undefined, translate values a small amount to apply the logarithm successfully
features_log_xformed[skewed] = features_imputed[skewed].apply(lambda x: np.log(x + 1))

fig = plt.figure(figsize = (15,10));

# Skewed feature plotting
for i, feature in enumerate(skewed):
    ax = fig.add_subplot(2, 3, i+1)
    sns.histplot(features_log_xformed[feature], bins=20, color='#00A0A0')
    ax.set_title("'%s' Feature Distribution"%(feature), fontsize = 14)
    ax.set_xlabel("Value")
    ax.set_ylabel("Number of Records")
    ax.set_ylim((0, 1600))
    ax.set_yticks([0, 400, 800, 1200, 1600])
    ax.set_yticklabels([0, 400, 800, 1200, ">1600"])

# Plot aesthetics
fig.suptitle("Log-Transformed Distributions of Continuous Data Features", \
        fontsize = 16, y = 1.03)

fig.tight_layout()

# Uncomment this line to save the figure.
# plt.savefig('../reports/figures/Log_Xformed_Distributions.svg')<jupyter_output><empty_output><jupyter_text>We also need to perform a log-transformation on our target variable 'SalePrice' to remove skew.<jupyter_code>target_log_xformed = target.transform(np.log)

# Uncomment this line to export DataFrame
# target_log_xformed.to_csv('../data/interim/target_log_xformed.csv')<jupyter_output><empty_output><jupyter_text>Feature Scaling Normalizing Numerical FeaturesIn addition to performing transformations on features that are highly skewed, it is often good practice to perform some type of scaling on numerical features. Applying a scaling to the data does not change the shape of each feature's distribution. Normalization does, however,  ensure that each feature is treated equally when applying supervised learners.<jupyter_code># Initialize scaler, then apply it to the features
scaler = StandardScaler()
numerical = features_log_xformed.select_dtypes(include=np.number).columns

features_scaled = pd.DataFrame(data = features_log_xformed)
features_scaled[numerical] = scaler.fit_transform(features_log_xformed[numerical])

features_final = features_scaled.copy()

# Uncomment this line to export DataFrame
features_scaled.to_csv('../data/interim/features_scaled.csv')

# Uncomment this line to export DataFrame
features_final.to_csv('../data/processed/features_final.csv')

# Show an example of a record with scaling applied
features_final.head()
# Stratified Shuffle Split - Come back to this later
# from sklearn.model_selection import StratifiedShuffleSplit
# ss = StratifiedShuffleSplit(test_size=0.2, random_state=5)

# i = 0
# for train_idx, test_idx in ss.split(np.zeros(features_final.shape[0]), features_final['OverallQual']):
#     X_train = features_final.iloc[train_idx]
#     y_train = saleprice.iloc[train_idx]
#     X_test = features_final.iloc[test_idx]
#     y_test = saleprice.iloc[test_idx]

# # Show the results of the split
# print(f'Training set has {X_train.shape[0]} samples.')
# print(f'Testing set has {X_test.shape[0]} samples.')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12257890961262553, 'fraction_numerical': 0.015602582496413199, 'mean_word_length': 6.270534550195567, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '122436', 'license': 'MIT', 'n_tokens_mistral': 3998, 'n_tokens_neox': 3755, 'n_words': 947, 'path': 'notebooks/0.2.1-saw-data-preparation.ipynb', 'repo_name': 'sward8080/housing-comp', 'size': '11152'}