<jupyter_start><jupyter_text>Feature Engineering In this approach we are going to do the below steps- Missing values- Temporal values- Categorical variables: remove rare labels- Standarise the values of the variables to the same range  Step 1: Getting all the missing values<jupyter_code>dataset = pd.read_csv("../data/train.csv")
dataset.shape
features_nan = [feature for feature in dataset.columns if dataset[feature].isnull().sum()>1 and dataset[feature].dtype=='O']
len(features_nan)
# Getting the missing value percentage
for feature in features_nan:
    print("{} --> {}".format(feature, np.round(dataset[feature].isnull().mean()*100, 2)))
# Heatmap of missing values
missingno.matrix(dataset[features_nan])<jupyter_output><empty_output><jupyter_text>step 1.1 Handling categorical varibles<jupyter_code># Function to replace missing categorical feature into a new label
def replace_cat_feature(dataset, columns):
    data = dataset.copy()
    data[columns] = data[columns].fillna('Missing')
    return data
dataset = replace_cat_feature(dataset, features_nan)
dataset.head()
# Has no missing values
missingno.matrix(dataset[features_nan])<jupyter_output><empty_output><jupyter_text>step 1.2 Handling numerical varibles<jupyter_code># Getting all the numerical values
numerical_with_nan = [feature for feature in dataset.columns if dataset[feature].isnull().sum() > 1 and dataset[feature].dtype != "O"]

for feature in numerical_with_nan:
    print("{} -> {}".format(feature, np.round(dataset[feature].isnull().mean()*100, 4)))
# Replacing the null values with the median of the column
for feature in numerical_with_nan:
    median_value = dataset[feature].median()
    dataset[feature + '_nan'] = np.where(dataset[feature].isnull(), 1, 0)
    dataset[feature].fillna(median_value, inplace=True)
# Getting all the nan values post transformation
print(dataset[numerical_with_nan].isnull().sum())
dataset.head()
# Getting the nan flag
for feature in numerical_with_nan:
    print(dataset[feature + "_nan"].value_counts())<jupyter_output>0    1201
1     259
Name: LotFrontage_nan, dtype: int64
0    1452
1       8
Name: MasVnrArea_nan, dtype: int64
0    1379
1      81
Name: GarageYrBlt_nan, dtype: int64<jupyter_text>step 1.3 Handling temporal varibles<jupyter_code># Transforming the year feature
for feature in ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']:
    dataset[feature] = dataset['YrSold'] - dataset[feature]
dataset[['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']].head()<jupyter_output><empty_output><jupyter_text>Step 2: Handling skewed numerical variablesSince all the numerical variables are skewed we are going to perform the log normal distribution<jupyter_code># Viz before log transformation
num_features = ['LotFrontage', 'LotArea', '1stFlrSF', 'GrLivArea', 'SalePrice']
for feature in num_features:
    sns.histplot(data=dataset, x=feature, kde=True)
    plt.show()
for feature in num_features:
    dataset[feature] = np.log(dataset[feature])
# Viz. all the features after log transformation
for feature in num_features:
    sns.histplot(data=dataset, x=feature, kde=True)
    plt.show()<jupyter_output><empty_output><jupyter_text>Step 3: Handling rare categorical features<jupyter_code>categorical_features = [feature for feature in dataset.columns if dataset[feature].dtype == 'O']
len(categorical_features)
categorical_features
for feature in categorical_features:
    temp = dataset.groupby(feature)['SalePrice'].count()/len(dataset)
    temp_df = temp[temp>0.01].index
    dataset[feature] = np.where(dataset[feature].isin(temp_df),dataset[feature], 'Rare_var')
dataset.head()<jupyter_output><empty_output><jupyter_text>step 4: Featue Scaling<jupyter_code>feature_scaling = [feature for feature in dataset.columns if feature not in ['Id', 'SalePrice']]
len(feature_scaling)
feature_scaling
scaler = MinMaxScaler()
scaler.fit_transform(dataset[feature_scaling])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11491317671092952, 'fraction_numerical': 0.014300306435137897, 'mean_word_length': 6.576402321083172, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '294229', 'license': 'MIT', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1217, 'n_words': 321, 'path': 'house-prices-reg-techniques/e2e/feature-engineering.ipynb', 'repo_name': 'tejeshreddy/kaggle-challenges', 'size': '3916'}