<jupyter_start><jupyter_text>Data Exploration<jupyter_code>#checking bedroom values
df['Bedrooms'].value_counts()
df['Bathrooms'].value_counts()
df['Furnishing'].value_counts()
df['Tennants'].value_counts()
df['Area'].value_counts()
df['Area'].unique<jupyter_output><empty_output><jupyter_text>Data cleansing<jupyter_code>def House_data_preprocessing(no_houses,mean_area,df):
    for house in range(no_houses): 
        
        # Preprocessing Price Data Values
        df.Price[house] = re.sub(r'\t','', df.Price[house])
        df.Price[house] = re.sub(r'\s','',df.Price[house])



        # Preprocessing Bathroom data Values
        if type(df.Bathrooms[house]) == float :
            df.Bathrooms[house] = '2'
            df.Bathrooms[house] = re.sub(r'\t','', df.Bathrooms[house])
            df.Bathrooms[house] = re.sub(r'\s+','',df.Bathrooms[house])  
        if df.Bathrooms[house][0] not in ['1','2','3','4','5']:
            df.Bathrooms[house] = '2'
        else : 
            df.Bathrooms[house] = df.Bathrooms[house][0]
            
        # Preprocessing Furnishing-type Values
        if df.Furnishing[house] in [ 'Bachelors/Family' , 'Family','Immediately' ] :
            df.Furnishing[house] = 'Semi-Furnished'
                # Preprocessing Tennants-Type Values
        if df.Tennants[house] not in ['Bachelors/Family', 'Family', 'Bachelors' ] :
            df.Tennants[house] = 'Bachelors/Family'
            
        # Preprocessing Area Values
        if 'sqft' not in df.Area[house]:
            df.Area[house] = '1477 sqft'
            
        # Preprocessing the Localtiy Values
        df.Locality[house] = re.sub(r'\t','',df.Locality[house])
        df.Locality[house] = re.sub(r'\s+','',df.Locality[house])  
        
    
    return df
def Mean_area_houses(no_houses):
    area = []
    for house in range(no_houses) :
        if type(df.Area[house]) == float :
            df.Area[house] = 'readmore'
        if 'sqft' in df.Area[house]:
            area_ = re.findall(r'\d+', df.Area[house])
            area.append(int(area_[0]))
    mean_area = mean(area)
    return(mean_area)
no_houses = len(df)
mean_area = Mean_area_houses(no_houses)
Housedata = House_data_preprocessing(no_houses, mean_area, df)
Housedata.head()
#Now checking only bedroom data
Housedata['Bedrooms'].unique()
#removing tabs and whitespaces
Housedata['Bedrooms']=Housedata['Bedrooms'].str.strip()
Housedata['Bedrooms'].unique()
 #As we can see there are some irrelevant data like studio apartment etc
def bedroom_process(no_houses,Housedata):
    for house in range(no_houses):
        if Housedata.Bedrooms[house][0] not in ['1','2','3','4','5']:
            Housedata.Bedrooms[house] = '3 BHK Apartment'
    return Housedata
no_houses = len(df)
housedata=bedroom_process(no_houses,Housedata)

housedata.head()
housedata['Bedrooms'].unique()
#creating a new column 'bhk'
housedata['bhk']=housedata['Bedrooms'].apply(lambda x: int(x.split(' ')[0]))
housedata.head()
Houserent=housedata.drop('Unnamed: 0',axis=1)
Houserent['bhk'].unique()
#Lets check locality
len(Houserent['Locality'])
len(Houserent['Locality'].unique())
#checking the data count of each locality data
locality_stats=Houserent.groupby('Locality')['Locality'].agg('count').sort_values(ascending=False)
locality_stats
len(locality_stats[locality_stats<=5])
locality_stats_less_than_5=locality_stats[locality_stats<=5]
locality_stats_less_than_5
Houserent.Locality.unique()
Houserent.Locality=Houserent.Locality.apply(lambda x: 'other' if x in locality_stats_less_than_5 else x )
len(Houserent['Locality'].unique())
Houserent['Locality'].unique()<jupyter_output><empty_output><jupyter_text>Data Visualisation<jupyter_code>for col in ['Furnishing','Tennants','bhk']:
    plt.figure(figsize=(11,7))
    sns.countplot(data =Houserent,x = col)
    plt.show()<jupyter_output><empty_output><jupyter_text>Model Building<jupyter_code>Houserent.head()
#Getting dummy variables for locality
dummies=pd.get_dummies(Houserent.Locality)
dummies.head()
hrent=pd.concat([Houserent,dummies.drop('other',axis='columns')],axis='columns')
hrent.head()
hrenters=hrent.drop('Locality',axis='columns')
hrenters.head()
hrenters=hrenters.drop('Bedrooms',axis='columns')
hrenters.head()
#Getting dummy variables for Furnishing
dummies=pd.get_dummies(hrenters.Furnishing)
dummies.head()
df5=pd.concat([hrenters,dummies.drop('Unfurnished',axis='columns')],axis='columns')
df5.head()
df6=df5.drop('Furnishing',axis='columns')
df6.head()
#Creating dummy variables for Tennants
dummies=pd.get_dummies(df6.Tennants)
dummies.head()
df7=pd.concat([df6,dummies.drop('Bachelors',axis='columns')],axis='columns')
df7.head()
#removing tenants column
df8=df7.drop('Tennants',axis='columns')
df8.head()
df8['Area']=df8['Area'].str.rstrip('sqft')
df8.head()
df8['Price']=df8['Price'].str.replace(',','').astype(float)
df8.head()
df8['Area']=df8['Area'].str.replace(',','').astype(float)
df8['Bathrooms']=df8['Bathrooms'].astype(int)
df8.head()
df8.shape
df8.head(3)
X=df8.drop('Price',axis='columns')
y=df8.Price
X.shape
y.shape
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=42)
X_train.shape
y_train.shape
X_test.shape
y_test.shape
import xgboost
from xgboost import XGBRegressor

my_model = XGBRegressor()
my_model = XGBRegressor(n_estimators=150,learning_rate=0.07)
my_model.fit(X_train,y_train, early_stopping_rounds=5, 
             eval_set=[(X_test,y_test)], verbose=False)
my_model.score(X_test,y_test)
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import accuracy_score,confusion_matrix
regressor = DecisionTreeRegressor(random_state = 0)
regressor.fit(X_train, y_train)
regressor.score(X_train,y_train)
X.columns
def predict_price(Locality,Furnishing,Tennants,Bathrooms,Area,bhk):    
    loc_index = np.where(X.columns==Locality)[0][0]
    fur_index = np.where(X.columns==Furnishing)[0][0]
    ten_index = np.where(X.columns==Tennants)[0][0]
    x = np.zeros(len(X.columns))
    x[0] = Bathrooms
    x[1] = Area
    x[2] = bhk
    if loc_index >= 0 & fur_index >= 0 & ten_index >= 0:
        x[loc_index] = 1
        x[fur_index] = 2
        x[ten_index] = 3
        

    return regressor.predict([x])[0]
predict_price('Kondapur','Semi-Furnished','Family',3, 1200, 2)
predict_price('Kondapur','Furnished','Family',3, 1200, 3)
#Exporting the model to pickle file
import pickle
with open('Hyderabad_house_rent.pickle','wb') as f:
    pickle.dump(regressor,f)
#Export location and column information to a file that will be useful later on in our prediction application
import json
columns = {
    'data_columns' : [col.lower() for col in X.columns]
}
with open("columns.json","w") as f:
    f.write(json.dumps(columns))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1561764705882353, 'fraction_numerical': 0.014705882352941176, 'mean_word_length': 4.955341506129598, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '33719', 'license': 'MIT', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2448, 'n_words': 396, 'path': 'Hyd_house_rent.ipynb', 'repo_name': 'saiteja6006/Hyderabad_houserent_prediction', 'size': '6800'}