<jupyter_start><jupyter_text><jupyter_code># --------------------------------------------------------------
# Simple Linear Regression for house prices 
# The dataset was obtained from https://github.com/Shreyas3108/house-price-prediction
# --------------------------------------------------------------
# ----------------------------------------------
# Step 0 - Import Libraries
# ---------------------------------------------
from google.colab import drive 
drive.mount('/content/drive/')
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import math
from sklearn import datasets
# ----------------------------------------------
# Step 1 - Read the data
# ----------------------------------------------
houses = pd.read_csv('/content/drive/My Drive/kc_house_data.csv')
houses.head()
# Show the columns names and the datatype they have.
houses.dtypes

# ----------------------------------------------
# Step 2 - Prelim Analysis and Feature selection
# ----------------------------------------------
# In order to save the orignal copy of the datafarme, we copy it to a new dataframe called houses_prep to start process and analyse the data.
houses_prep = houses.copy()
# we drop the columns of date and id because it will not affect the prediction.
# we drop the column of zipcode even we can use the first digits as an area indicators.
houses_prep = houses_prep.drop(['date', 'id', 'zipcode'], axis=1)
# we check if there is any missing value.
houses_prep.isnull().sum()
# Create pandas histogram to check if the price is normal distribution
houses_prep.hist(column= 'price', rwidth = 0.9)<jupyter_output><empty_output><jupyter_text>The histogram shows that price variable is not Normally Distributed<jupyter_code># ----------------------------------------------
# Step 3 - Data Visualisation
# ----------------------------------------------
#price            float64
# Visualise the continuous features Vs price
# The continuous features are sqft_living , sqft_lot, sqft_above , sqft_basement , lat , long  , sqft_living15 , sqft_lot15.
fig = plt.figure(figsize=(8, 8))
plt.subplot(4,2,1) 
plt.title('sqft_living Vs price')
plt.scatter(houses_prep['sqft_living'], houses_prep['price'], s=2, c='g')
plt.tight_layout()

plt.subplot(4,2,2) 
plt.title('sqft_lot Vs price')
plt.scatter(houses_prep['sqft_lot'], houses_prep['price'], s=2, c='r')
plt.tight_layout()


plt.subplot(4,2,3) 
plt.title('sqft_above Vs price')
plt.scatter(houses_prep['sqft_above'], houses_prep['price'], s=2, c='b')
plt.tight_layout()

plt.subplot(4,2,4) 
plt.title('sqft_basement Vs price')
plt.scatter(houses_prep['sqft_basement'], houses_prep['price'], s=2, c='y')
plt.tight_layout()

plt.subplot(4,2,5) 
plt.title('lat Vs price')
plt.scatter(houses_prep['lat'], houses_prep['price'], s=2, c='b')
plt.tight_layout()

plt.subplot(4,2,6) 
plt.title(' long Vs price')
plt.scatter(houses_prep['long'], houses_prep['price'], s=2, c='y')
plt.tight_layout()

plt.subplot(4,2,7) 
plt.title(' sqft_living15  Vs price')
plt.scatter(houses_prep['sqft_living15'], houses_prep['price'], s=2, c='r')
plt.tight_layout()


plt.subplot(4,2,8) 
plt.title('sqft_lot15 Vs price')
plt.scatter(houses_prep['sqft_lot15'], houses_prep['price'], s=1, c='r')
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>**The scatter plots show that:**
1.   The price increase while sqft_living increase
2.   The price increase while sqft_above increase.
3.   The price increase while sqft_living15 increase.
4.   The graphes of sqft_living, sqft_above, sqft_living15 are very similar.<jupyter_code># Visualise the categorical features
colors = ['g', 'r', 'm', 'b']
fig = plt.figure(figsize=(8, 8))
plt.subplot(4,2,1)
plt.title('Average Price vs bedrooms')
cat_list = houses_prep['bedrooms'].unique()
cat_average = houses_prep.groupby('bedrooms').mean()['price']
plt.bar(cat_list, cat_average, color=colors)
plt.tight_layout()

plt.subplot(4,2,2)
plt.title('Average Price vs bathrooms')
cat_list = houses_prep['bathrooms'].unique()
cat_average = houses_prep.groupby('bathrooms').mean()['price']
plt.bar(cat_list, cat_average, color=colors)
plt.tight_layout()

plt.subplot(4,2,3)
plt.title('Average Price vs floors')
cat_list = houses_prep['floors'].unique()
cat_average = houses_prep.groupby('floors').mean()['price']
plt.bar(cat_list, cat_average, color=colors)
plt.tight_layout()

plt.subplot(4,2,4)
plt.title('Average Price vs waterfront')
cat_list = houses_prep['waterfront'].unique()
cat_average = houses_prep.groupby('waterfront').mean()['price']
plt.bar(cat_list, cat_average, color=colors)
plt.tight_layout()


plt.subplot(4,2,5)
plt.title('Average Price vs condition')
cat_list = houses_prep['condition'].unique()
cat_average = houses_prep.groupby('condition').mean()['price']
plt.bar(cat_list, cat_average, color=colors)
plt.tight_layout()

plt.subplot(4,2,6)
plt.title('Average Price vs yr_built')
cat_list = houses_prep['yr_built'].unique()
cat_average = houses_prep.groupby('yr_built').mean()['price']
plt.bar(cat_list, cat_average, color=colors)
plt.tight_layout()

plt.subplot(4,2,7)
plt.title('Average Price vs yr_renovated')
cat_list = houses_prep['yr_renovated'].unique()
cat_average = houses_prep.groupby('yr_renovated').mean()['price']
plt.bar(cat_list, cat_average, color=colors)
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>There are variations between categories of each graph, so we cannot decide that deleting some categorical features will improve the model or not<jupyter_code># Check for outliers
houses_prep['price'].describe().apply("{0:.5f}".format)<jupyter_output><empty_output><jupyter_text>50% of the prices are between 321950 and 645000.
The differences between the mean and the max is too large.<jupyter_code>houses_prep['price'].quantile([0.05, 0.1, 0.15, 0.5, 0.9, 0.95, 0.99])<jupyter_output><empty_output><jupyter_text>It is now clear thatr there are Outliers (the min and max) in the data.<jupyter_code># ------------------------------------------------------
# Step 4 - Check Multiple Linear Regression Assumptions
# ------------------------------------------------------
# Linearity using correlation coefficient matrix using corr
pd.options.display.max_columns = None
correlation = houses_prep[['sqft_living', 'sqft_above', 'lat', 'sqft_basement', 'long', 'sqft_living15', 'price']].corr()
print(correlation)<jupyter_output>sqft_living  sqft_above       lat  sqft_basement      long  \
sqft_living       1.000000    0.876597  0.052529       0.435043  0.240223   
sqft_above        0.876597    1.000000 -0.000816      -0.051943  0.343803   
lat               0.052529   -0.000816  1.000000       0.110538 -0.135512   
sqft_basement     0.435043   -0.051943  0.110538       1.000000 -0.144765   
long              0.240223    0.343803 -0.135512      -0.144765  1.000000   
sqft_living15     0.756420    0.731870  0.048858       0.200355  0.334605   
price             0.702035    0.605567  0.307003       0.323816  0.021626   

               sqft_living15     price  
sqft_living         0.756420  0.702035  
sqft_above          0.731870  0.605567  
lat                 0.048858  0.307003  
sqft_basement       0.200355  0.323816  
long                0.334605  0.021626  
sqft_living15       1.000000  0.585379  
price               0.585379  1.000000<jupyter_text>we are going to drop (long ) becuase it does not have correlation with the price.

we are also going to drop the sqft_above because its correlation with sqft_living is high so one variable is enough.<jupyter_code># Step 5 - Drop irrelevant features
houses_prep = houses_prep.drop(['sqft_above', 'long'], axis=1)
houses_prep.head()
# ------------------------------------------------------
# Step 6 - Create/Modify new features
# ------------------------------------------------------
#The price is currently log normal distribution, we want to change it to normal distribution in the next step.
df1= houses_prep['price']
df2= np.log(df1)

plt.figure()
df1.hist(rwidth=0.9, bins=20)

plt.figure()
df2.hist(rwidth=0.9, bins=20)
houses_prep['price']= np.log(houses_prep['price'])
# -----------------------------------------------------------------------------
# Step 7 - Create Dummy Variables and drop first to avoid dummy variables trap
# -----------------------------------------------------------------------------
# - bedrooms         0
# - bathrooms 
# - floors
# - waterfront
# - view
# - condition
# - grade
# - yr_built
# - yr_renovated
# Using get_dummies

houses_prep['bedrooms'] = houses_prep['bedrooms'].astype('category')
houses_prep['bathrooms'] = houses_prep['bathrooms'].astype('category')
houses_prep['floors'] = houses_prep['floors'].astype('category')
houses_prep['waterfront'] = houses_prep['waterfront'].astype('category')
houses_prep['view'] = houses_prep['view'].astype('category')
houses_prep['condition'] = houses_prep['condition'].astype('category')
houses_prep['grade'] = houses_prep['grade'].astype('category')
houses_prep['yr_built'] = houses_prep['yr_built'].astype('category')
houses_prep['yr_renovated'] = houses_prep['yr_renovated'].astype('category')
houses_prep = pd.get_dummies(houses_prep, drop_first=True)


# ----------------------------------------------------------
# Step 8 - Create Train and test split
# ----------------------------------------------------------

# Split the X and Y dataset into training and testing set
#
#      from sklearn.model_selection import train_test_split
#      X_train, X_test, Y_train, Y_test = \
#      train_test_split(X, Y, test_size = 0.4, random_state = 1234)
#
# Demand is time-series type of data

Y = houses_prep[['price']]
X = houses_prep.drop(['price'], axis= 1)

tr_size = 0.7 * len(X)
tr_size = int(tr_size)

X_train = X.values[0 : tr_size]
X_test = X.values[tr_size : len(X)]

Y_train = Y.values[0 : tr_size]
Y_test = Y.values[tr_size : len(Y)]


# ----------------------------------------------------------
# Step 9 - Fit and Score the model
# ----------------------------------------------------------
# Linear Regression
from sklearn.linear_model import LinearRegression

std_reg = LinearRegression()
std_reg.fit(X_train, Y_train)

r2_train = std_reg.score(X_train, Y_train)
r2_test  = std_reg.score(X_test, Y_test)


# Create Y Predictions
Y_predict = std_reg.predict(X_test)


from sklearn.metrics import mean_squared_error
rmse = math.sqrt(mean_squared_error(Y_test, Y_predict))

print('r2_test accuracy', r2_test)
print('r2_train accuracy', r2_train)
print('RMSE', rmse)


# Calculate RMSLE and compare results
# Exponent opposite to log

Y_test_e = []
Y_predict_e = []

for i in range(0, len(Y_test)):
    Y_test_e.append(math.exp(Y_test[i]))
    Y_predict_e.append(math.exp(Y_predict[i]))

log_sq_sum = 0.0

for i in range(0, len(Y_test_e)):
    log_a = math.log(Y_test_e[i] + 1)
    log_p = math.log(Y_predict_e[i] + 1)
    log_diff = (log_p - log_a)**2
    log_sq_sum = log_sq_sum + log_diff

rmsle = math.sqrt(log_sq_sum/len(Y_test))

print('RMSLE:', rmsle)<jupyter_output>RMSLE: 0.2491717795823871
{'fraction_non_alphanumeric': 0.21375812049479398, 'fraction_numerical': 0.04778855566432322, 'mean_word_length': 2.7224246439218285, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '600552', 'license': 'MIT', 'n_tokens_mistral': 4231, 'n_tokens_neox': 3728, 'n_words': 835, 'path': 'Linear_Regression_Houses.ipynb', 'repo_name': 'baharababah/Linear_Regression_StepByStep', 'size': '11237'}