<jupyter_start><jupyter_text>Data cleaning<jupyter_code>import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
%matplotlib inline
import matplotlib
matplotlib.rcParams["figure.figsize"] = (20, 10)
df1 = pd.read_csv("../data/raw/Bengaluru_House_Data.csv")
df1.head()
df1.groupby('area_type')['area_type'].agg('count')
df2 = df1.drop(['area_type', 'society', 'balcony', 'availability'], axis='columns')
df2.head()
df2.isnull().sum()
df3 = df2.dropna()
df3.isnull().sum()
df3.shape
df3['size'].unique()
df3['bhk'] = df3['size'].apply(lambda x: int(x.split()[0]))
df3.head()
df3[df3.bhk>20]
df3.total_sqft.unique()
def isfloat(x):
    try:
        float(x)
    except:
        return False
    return True
df3[~df3['total_sqft'].apply(isfloat)].head(10)
def convert_sqft_to_num(x):
    tokens = x.split('-')
    if len(tokens) == 2:
        return (float(tokens[0])+float(tokens[1]))/2
    try:
        return float(x)
    except:
        return None
df4 = df3.copy()
df4['total_sqft'] = df4['total_sqft'].apply(convert_sqft_to_num)
df4.head()
df4.loc[30]<jupyter_output><empty_output><jupyter_text>Feature engineering<jupyter_code>df5 = df4.copy()
df5['price_per_sqft'] = (df5['price'] / df5['total_sqft']) * 100000
# multiplication id for converting to LAKH
df5.head()
len(df5.location.unique())<jupyter_output><empty_output><jupyter_text>1304 locations is too much and will cause dimentionality curse when doing onehotencoding<jupyter_code>df5.location = df5.location.apply(lambda x: x.strip())

location_stats = df5.groupby('location')['location'].agg('count').sort_values(ascending=False)
location_stats
len(location_stats[location_stats<=10])
location_stats_less_than_10 = location_stats[location_stats<=10]
location_stats_less_than_10
df5.location = df5.location.apply(lambda x: 'other' if x in location_stats_less_than_10 else x)
len(df5.location.unique())
df5.head()<jupyter_output><empty_output><jupyter_text>Outliers removal<jupyter_code>df5[df5.total_sqft/df5.bhk<300].head()
df5.shape
df6 = df5[~(df5.total_sqft/df5.bhk<300)]
df6.shape
df6.price_per_sqft.describe()
def remove_pps_outliers(df):
    df_out = pd.DataFrame()
    for key, subdf in df.groupby('location'):
        m = np.mean(subdf.price_per_sqft)
        st = np.std(subdf.price_per_sqft)
        reduced_df = subdf[(subdf.price_per_sqft>=(m-st)) & (subdf.price_per_sqft<=(m+st))]
        df_out = pd.concat([df_out, reduced_df], ignore_index=True)
    return df_out 

df7 = remove_pps_outliers(df6)
df7.shape
def plot_scatter_chart(df, location):
    bhk2 = df[(df.location == location) & (df.bhk==2)]
    bhk3 = df[(df.location == location) & (df.bhk==3)]

    matplotlib.rcParams['figure.figsize'] = (15,10)

    plt.scatter(bhk2.total_sqft, bhk2.price_per_sqft, color='blue', label='2 BHK', s=50)
    plt.scatter(bhk3.total_sqft, bhk3.price_per_sqft, color='green', marker='+', label='3 BHK', s=50)
    
    plt.xlabel("Total Square Feet Area")
    plt.ylabel("Price Per Square Feet")
    plt.title(location)

    plt.legend()


plot_scatter_chart(df7, "<NAME>")<jupyter_output><empty_output><jupyter_text>**We should also remove properties where for same location, the price of (for example) 3 bedroom apartment is less than 2 bedroom apartment (with same square ft area). What we will do is for a given location, we will build a dictionary of stats per bhk, i.e.**{    '1' : {        'mean': 4000,        'std: 2000,        'count': 34    },    '2' : {        'mean': 4300,        'std: 2300,        'count': 22    },    }**Now we can remove those 2 BHK apartments whose price_per_sqft is less than mean price_per_sqft of 1 BHK apartment**<jupyter_code>def remove_bhk_outliers(df):
    exclude_indices = np.array([])
    for location, location_df in df.groupby('location'):
        bhk_stats = {}
        for bhk, bhk_df in location_df.groupby('bhk'):
            bhk_stats[bhk] = {
                'mean': np.mean(bhk_df.price_per_sqft),
                'std': np.std(bhk_df.price_per_sqft),
                'count': bhk_df.shape[0]
            }
        for bhk, bhk_df in location_df.groupby('bhk'):
            stats = bhk_stats.get(bhk-1)
            if stats and stats['count']>5:
                exclude_indices = np.append(exclude_indices, bhk_df[bhk_df.price_per_sqft<(stats['mean'])].index.values)
    return df.drop(exclude_indices,axis='index')
    
df8 = remove_bhk_outliers(df7)
df8.shape
plot_scatter_chart(df8,"<NAME>")
matplotlib.rcParams["figure.figsize"] = (20,10)
plt.hist(df8.price_per_sqft,rwidth=0.8)
plt.xlabel("Price Per Square Feet")
plt.ylabel("Count")<jupyter_output><empty_output><jupyter_text>Outlier Removal Using Bathrooms Feature<jupyter_code>df8.bath.unique()
plt.hist(df8.bath,rwidth=0.8)
plt.xlabel("Number of bathrooms")
plt.ylabel("Count")
df8[df8.bath>10]<jupyter_output><empty_output><jupyter_text>It is unusual to have 2 more bathrooms than number of bedrooms in a home<jupyter_code>df8[df8.bath>df8.bhk+2]<jupyter_output><empty_output><jupyter_text>the business manager has a conversation with you (i.e. a data scientist) that if you have 4 bedroom home and even if you have bathroom in all 4 rooms plus one guest bathroom, you will have total bath = total bed + 1 max. Anything above that is an outlier or a data error and can be removed<jupyter_code>df9 = df8[df8.bath<df8.bhk+2]
df9.shape
df10 = df9.drop(['size','price_per_sqft'],axis='columns')
df10.head(3)<jupyter_output><empty_output><jupyter_text>Model building  One Hot Encoding<jupyter_code>dummies = pd.get_dummies(df10.location)
dummies.head(3)
# concatenation and escaping dummy trap
df11 = pd.concat([df10, dummies.drop('other', axis='columns')], axis='columns')
df12 = df11.drop('location', axis='columns')
df12.head(3)
df12.shape<jupyter_output><empty_output><jupyter_text>Splitting data<jupyter_code>X = df12.drop('price', axis="columns")
y = df12.price
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=10)<jupyter_output><empty_output><jupyter_text>Training & evaluation<jupyter_code>from sklearn.linear_model import LinearRegression

lr_clf = LinearRegression()
lr_clf.fit(X_train, y_train)
lr_clf.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Cross Validation<jupyter_code>from sklearn.model_selection import ShuffleSplit, cross_val_score

cv = ShuffleSplit(n_splits=5, test_size=0.2, random_state=0)
cross_val_score(LinearRegression(), X, y, cv=cv)<jupyter_output><empty_output><jupyter_text>Hyperparameter tuning<jupyter_code>from sklearn.model_selection import GridSearchCV

from sklearn.linear_model import Lasso
from sklearn.tree import DecisionTreeRegressor

def find_best_model_using_gridsearchcv(X, y):
    algos = {
        'linear_regression': {
            'model': LinearRegression(),
            'params': {
                'normalize': [True, False]
            }
        },
        'lasso': {
            'model': Lasso(),
            'params': {
                'alpha': [1, 2],
                'selection': ['random', 'cyclic']
            }
        },
        'decision_tree': {
            'model': DecisionTreeRegressor(),
            'params': {
                'criterion': ['mse', 'friedman_mse'],
                'splitter': ['best', 'random']
            }
        }
    }

    scores = []
    cv = ShuffleSplit(n_splits=5, test_size=0.2, random_state=0)
    for algo_name, config in algos.items():
        gs =  GridSearchCV(config['model'], config['params'], cv=cv, return_train_score=False)
        gs.fit(X,y)
        scores.append({
            'model': algo_name,
            'best_score': gs.best_score_,
            'best_params': gs.best_params_
        })

    return pd.DataFrame(scores,columns=['model','best_score','best_params'])

find_best_model_using_gridsearchcv(X,y)<jupyter_output>/home/amin/miniconda3/lib/python3.9/site-packages/sklearn/linear_model/_base.py:141: FutureWarning: 'normalize' was deprecated in version 1.0 and will be removed in 1.2.
If you wish to scale the data, use Pipeline with a StandardScaler in a preprocessing stage. To reproduce the previous behavior:

from sklearn.pipeline import make_pipeline

model = make_pipeline(StandardScaler(with_mean=False), LinearRegression())

If you wish to pass a sample_weight parameter, you need to pass it as a fit parameter to each step of the pipeline as follows:

kwargs = {s[0] + '__sample_weight': sample_weight for s in model.steps}
model.fit(X, y, **kwargs)


  warnings.warn(
/home/amin/miniconda3/lib/python3.9/site-packages/sklearn/linear_model/_base.py:141: FutureWarning: 'normalize' was deprecated in version 1.0 and will be removed in 1.2.
If you wish to scale the data, use Pipeline with a StandardScaler in a preprocessing stage. To reproduce the previous behavior:

from sklearn.pipeline import make_pip[...]<jupyter_text>Based on above results we can say that LinearRegression gives the best score. Hence we will use that.  Making predictions<jupyter_code>def predict_price(location,sqft,bath,bhk):    
    loc_index = np.where(X.columns==location)[0][0]

    x = np.zeros(len(X.columns))
    x[0] = sqft
    x[1] = bath
    x[2] = bhk
    if loc_index >= 0:
        x[loc_index] = 1

    return lr_clf.predict([x])[0]
predict_price('1st Phase JP Nagar',1000, 2, 2)<jupyter_output>/home/amin/miniconda3/lib/python3.9/site-packages/sklearn/base.py:450: UserWarning: X does not have valid feature names, but LinearRegression was fitted with feature names
  warnings.warn(<jupyter_text>Exporting the model<jupyter_code>import pickle
with open('home_prices_model.pickle','wb') as f:
    pickle.dump(lr_clf,f)
! mv ./home_prices_model.pickle ../data/model/home_prices_model.pickle<jupyter_output><empty_output><jupyter_text>Export location and column information to a file that will be useful later on in our prediction application<jupyter_code>import json
columns = {
    'data_columns' : [col.lower() for col in X.columns]
}
with open("../data/cols/columns.json","w") as f:
    f.write(json.dumps(columns))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13177217701924984, 'fraction_numerical': 0.025501091486406034, 'mean_word_length': 4.392723381487427, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '616961', 'license': 'MIT', 'n_tokens_mistral': 3750, 'n_tokens_neox': 3569, 'n_words': 753, 'path': 'ml/model.ipynb', 'repo_name': 'aminbenmansour/real-estate-price-prediction', 'size': '10090'}