<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Load dataset<jupyter_code>df = pd.read_csv('final_house_scrape.csv')
df.head(10)<jupyter_output><empty_output><jupyter_text>select the dependent and independent features for prediction.<jupyter_code>X = df.drop("price(L)", axis=1)
y = df['price(L)']
print('Shape of X = ', X.shape)
print('Shape of y = ', y.shape)<jupyter_output>Shape of X =  (81212, 154)
Shape of y =  (81212,)<jupyter_text>split the data into train and test<jupyter_code>from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 51)
print('Shape of X_train = ', X_train.shape)
print('Shape of y_train = ', y_train.shape)
print('Shape of X_test = ', X_test.shape)
print('Shape of y_test = ', y_test.shape)<jupyter_output>Shape of X_train =  (64969, 154)
Shape of y_train =  (64969,)
Shape of X_test =  (16243, 154)
Shape of y_test =  (16243,)<jupyter_text>Feature Scaling<jupyter_code>from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
sc.fit(X_train)
X_train= sc.transform(X_train)
X_test = sc.transform(X_test)<jupyter_output><empty_output><jupyter_text>Model Building  Linear Regression<jupyter_code>from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.metrics import mean_squared_error
lr = LinearRegression()
lr_lasso = Lasso()
def rmse(y_test, y_pred):
    return np.sqrt(mean_squared_error(y_test, y_pred))
lr.fit(X_train, y_train)
lr_score = lr.score(X_test, y_test) 
lr_rmse = rmse(y_test, lr.predict(X_test))
lr_score, lr_rmse<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code>lr_lasso.fit(X_train, y_train)
lr_lasso_score=lr_lasso.score(X_test, y_test) 
lr_lasso_rmse = rmse(y_test, lr_lasso.predict(X_test))
lr_lasso_score, lr_lasso_rmse<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble import RandomForestRegressor
rfr = RandomForestRegressor()
rfr.fit(X_train,y_train)
rfr_score=rfr.score(X_test,y_test) 
rfr_rmse = rmse(y_test, rfr.predict(X_test))
rfr_score, rfr_rmse<jupyter_output><empty_output><jupyter_text>Cross validation<jupyter_code>from sklearn.model_selection import cross_val_score

cvs_rfr = cross_val_score(rfr, X_train,y_train, cv = 10)
cvs_rfr, cvs_rfr.mean()
cvs_rfr2 = cross_val_score(RandomForestRegressor(), X_train,y_train, cv = 10)
cvs_rfr2, cvs_rfr2.mean()<jupyter_output><empty_output><jupyter_text>Serialize Models<jupyter_code>import pickle
# open a file, where you ant to store the data
file = open('lasso_model.pkl', 'wb')

# dump information to that file
pickle.dump(lr_lasso, file)
import pickle
# open a file, where you ant to store the data
file = open('random_model.pkl', 'wb')

# dump information to that file
pickle.dump(rfr, file)
import pickle
# open a file, where you ant to store the data
file = open('linear_model.pkl', 'wb')

# dump information to that file
pickle.dump(lr, file)
list(X.columns)<jupyter_output><empty_output><jupyter_text>Method for Prediction<jupyter_code>def predict_house_price(model,BHK,Bathroom,rate_persqft,area_insqft,construction_status,Sale_type,location):
    x =np.zeros(len(X.columns)) # create zero numpy array, len = 107 as input value for model
    # adding feature's value accorind to their column index
    x[0]=BHK
    x[1]=Bathroom
    x[2]=rate_persqft
    x[3]=area_insqft
 
    if "construction_status"=="Ready To Move":
        x[5]=1
     
    if "Sale_type"=="resale":
        x[7]=1
 
    if 'location_'+location in X.columns:
        loc_index = np.where(X.columns=="location_"+location)[0][0]
        x[loc_index] =1
 
    #print(loc_index)
 
  #print(x)
 
  # feature scaling
    x = sc.transform([x])[0] # give 2d np array for feature scaling and get 1d scaled np array
  #print(x)
 
    return model.predict([x])[0] # return the predicted value by train Random forest model

#Test on Linear Part-1
predict_house_price(model=lr ,BHK=1,Bathroom=1,rate_persqft=14545,area_insqft=550,construction_status="Ready to move",Sale_type="resale",location="Ghatkopar") 
# Test on Linear part 2
predict_house_price(model=lr,BHK=1,Bathroom=2,rate_persqft=14222,area_insqft=675,construction_status="Under Construction",Sale_type="new",location="Jogeshwari West")
#Test on Lasso Part-1
predict_house_price(model=lr_lasso,BHK=1,Bathroom=1,rate_persqft=14545,area_insqft=550,construction_status="Ready to move",Sale_type="resale",location="Ghatkopar") 
# Test on lassso part 2
predict_house_price(model=lr_lasso,BHK=1,Bathroom=2,rate_persqft=14222,area_insqft=675,construction_status="Under Construction",Sale_type="new",location="Jogeshwari West")
# Test on Random forest part 1
predict_house_price(model=rfr,BHK=1,Bathroom=1,rate_persqft=14545,area_insqft=550,construction_status="Ready to move",Sale_type="resale",location="Ghatkopar")
# Test on Random forest part 2
predict_house_price(model=rfr,BHK=1,Bathroom=2,rate_persqft=14222,area_insqft=675,construction_status="Under Construction",Sale_type="new",location="Jogeshwari West")
# From the Above predictions we see that in Lasso model there is error in price value around 5-6 lacs from original price
#While RandomForest has negligible error compared to Lasso and Linear regression so we will use Random Forest for further user-interface
#Linear Regression model totally failed to predict the values.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11071042192571222, 'fraction_numerical': 0.025063108546700325, 'mean_word_length': 7.366515837104072, 'pattern_counts': {'":': 2, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '235069', 'license': 'MIT', 'n_tokens_mistral': 2182, 'n_tokens_neox': 2060, 'n_words': 425, 'path': 'Code Notebooks/mumbai_house_model.ipynb', 'repo_name': 'piyushr385/RealEstatePricePrediction', 'size': '5546'}