<jupyter_start><jupyter_text><jupyter_code>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import keras
from keras.models import Sequential, load_model
from keras.layers.core import Dense, Dropout, Activation
from keras.utils import np_utils
from keras.wrappers.scikit_learn import KerasRegressor

#from kerastuner.tuners import RandomSearch # HyperParameter Tunining
from keras.optimizers import Adam # Optimizer used in the NN
from keras.callbacks import EarlyStopping # Early Stopping Callback in the NN

import itertools

import pandas as pd
pd.set_option('display.max_rows', 100)
pd.set_option('display.max_columns', 100)
pd.set_option('display.width', 1000)

from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax

import numpy as np
import matplotlib.pyplot as plt
from pylab import rcParams
import matplotlib

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
train = pd.read_csv('/content/train.csv')
test = pd.read_csv('/content/test.csv')<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis  (EDA)<jupyter_code>train.drop(['Id'], axis=1, inplace=True)
test.drop(['Id'], axis=1, inplace=True)
# Log is used to respond to skewness towards large values 
train["SalePrice"] = np.log1p(train["SalePrice"])
y = train['SalePrice'].reset_index(drop=True)
corr_matrix = train.corr(method='pearson')
corr_matrix['SalePrice'].sort_values(kind="quicksort")
train["BsmtFinSF1"].hist(bins=30, figsize=(15,2))
train.boxplot(figsize=(15,7), column=[
'BsmtFinSF1',
'1stFlrSF',
'TotalBsmtSF',
'GrLivArea',])
train.boxplot(figsize=(15,7), column=[
'BsmtUnfSF',
'2ndFlrSF',
'MasVnrArea',
'GarageArea',
])
train[train['2ndFlrSF'] > 1500]
train.boxplot(figsize=(15,7), column=[
'OpenPorchSF',
'WoodDeckSF',
'LotFrontage',
])
train.boxplot(figsize=(15,7), column=[

'KitchenAbvGr',
'BsmtHalfBath',
'BedroomAbvGr',
'BsmtFullBath',
'HalfBath',
'Fireplaces',
'TotRmsAbvGrd',
'FullBath',
'GarageCars',
'OverallQual'
])
train[train['BsmtFullBath'] > 2]
train[train['OverallQual'] < 2]
train.boxplot(figsize=(15,5), column=[
'EnclosedPorch',
'3SsnPorch',
'ScreenPorch'
])
train.boxplot(figsize=(15,5), column=[
#'YrSold',
#'MoSold',
'YearRemodAdd',
'YearBuilt'
])
train[train['YearBuilt'] < 1900]
# Put test & train features to one df
train_features = train.drop(['SalePrice'], axis=1)
test_features = test
features = pd.concat([train_features, test_features]).reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Data Cleansing<jupyter_code># Eploring NaN

def missing_Values_table(df):
  mis_val = df.isnull().sum()
  mis_val_percent = 100 * df.isnull().sum() / len(df)
  mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
  mis_val_table_ren_columns = mis_val_table.rename(columns= {0: 'Missing Values', 1 : '% of Total Values'})
  mis_val_table_ren_columns = mis_val_table_ren_columns[
        mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
    '% of Total Values', ascending=False).round(1)
  return mis_val_table_ren_columns

missing_Values_table(features)
features = features.drop(['PoolQC','MiscFeature','Alley','Fence','FireplaceQu'], axis=1)
features['MSSubClass'] = features['MSSubClass'].apply(str)
features['YrSold'] = features['YrSold'].astype(str)
features['MoSold'] = features['MoSold'].astype(str)
print(features['Functional'].value_counts(dropna=False))
print(features['Electrical'].value_counts(dropna=False))
print(features['KitchenQual'].value_counts(dropna=False))
## Filling these columns With most suitable value for these columns 
features['Functional'] = features['Functional'].fillna('Typ') 
features['Electrical'] = features['Electrical'].fillna("SBrkr") 
features['KitchenQual'] = features['KitchenQual'].fillna("TA") 
print(features['Exterior1st'].value_counts(dropna=False))
print(features['Exterior2nd'].value_counts(dropna=False))
print(features['SaleType'].value_counts(dropna=False))
## Filling these with MODE, i.e. , the most frequent value in these columns .
features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode()[0]) 
features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode()[0])
features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode()[0])
### Missing data in GarageYrBit most probably means missing Garage, so replace NaN with zero .
for col in ('GarageYrBlt', 'GarageArea', 'GarageCars'):
    features[col] = features[col].fillna(0)
for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
    features[col] = features[col].fillna('None')

### Same with basement
for col in ('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
    features[col] = features[col].fillna('None')
missing_Values_table(features)
features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode()[0]))

# Fill the remaining columns as None
objects = []
for i in features.columns:
    if features[i].dtype == object:
        objects.append(i)
features.update(features[objects].fillna('None'))
missing_Values_table(features)
features["LotFrontage"].hist(bins=30, figsize=(15,3))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))

numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
    if features[i].dtype in numeric_dtypes:
        numerics.append(i)
features.update(features[numerics].fillna(0))


numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
    if features[i].dtype in numeric_dtypes:
        numerics2.append(i)
skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)

high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index

for i in skew_index:
    features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
missing_Values_table(features)
features<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>print(features['Utilities'].value_counts(dropna=False))
print(features['Street'].value_counts(dropna=False))
# Removing features that are not very useful:
# - droped because ... : 'Utilities', 'Street'
features = features.drop(['Utilities', 'Street'], axis=1)
print(features['MasVnrType'].value_counts(dropna=False))
print(features['MasVnrArea'].value_counts(dropna=False))

# Adding new features. Sums of categiries.

features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']

features['Total_sqr_footage'] = (features['BsmtFinSF1'] + features['BsmtFinSF2'] +
                                 features['1stFlrSF'] + features['2ndFlrSF'])

features['Total_Bathrooms'] = (features['FullBath'] + (0.5 * features['HalfBath']) +
                               features['BsmtFullBath'] + (0.5 * features['BsmtHalfBath']))

features['Total_porch_sf'] = (features['OpenPorchSF'] + features['3SsnPorch'] +
                              features['EnclosedPorch'] + features['ScreenPorch'] +
                              features['WoodDeckSF'])

## If PoolArea = 0 , Then HasPool = 0 too, ...

features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)

features.shape
features
final_features = pd.get_dummies(features).reset_index(drop=True)
final_features.shape
final_features
##########################################################
#  Now, again train and test are spilt back seperately,  #
#  as now all data processing is done.                   #
#  Y is taget and its length is used to split            #
##########################################################

X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(y):, :]
X.shape, y.shape, X_sub.shape
final_features.iloc[[30,88,462, 631, 1322]]
# Removing outliers.
# Can be seen by plotting them in a graph.

outliers = [30, 88, 462, 631, 1322]

# From RandomForest:
# outliers = [30, 39, 58, 88, 108, 307, 375, 462, 520, 523, 533, 631, 635, 636, 705, 769, 778, 828, 954, 1179, 1218, 1219, 1298, 1322, 1337]

X = X.drop(X.index[outliers])
y = y.drop(y.index[outliers])
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)<jupyter_output><empty_output><jupyter_text>Random Forest Regressor Model<jupyter_code>from sklearn.ensemble import RandomForestRegressor
regr = RandomForestRegressor(n_estimators=100, random_state=0)
regr.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Training Accuracy<jupyter_code>regr.score(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Testing Accuracy<jupyter_code>regr.score(X_test, y_test)
expected = y_test
predicted = regr.predict(X_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1577680067001675, 'fraction_numerical': 0.027324120603015076, 'mean_word_length': 5.942587209302325, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '465061', 'license': 'MIT', 'n_tokens_mistral': 3600, 'n_tokens_neox': 3352, 'n_words': 633, 'path': 'Housing_Price_Random_Forest_Regressor.ipynb', 'repo_name': 'vdnew/House-Prices---Advanced-Regression-Techniques', 'size': '9552'}