<jupyter_start><jupyter_text>Prospecting LA's Backyard Houses with Machine Learning     <NAME>   Nightly Price Predictions for Backyard Houses<jupyter_code>import pandas as pd
import numpy as np

import seaborn as sns
sns.set()

#plotting
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import RidgeCV
%matplotlib inline

#preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn import metrics
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import GradientBoostingRegressor
import xgboost as xgb

from statistics import mean

pd.set_option('display.max_row', 1000)
pd.set_option('display.max_column', 100)

#get pandas and sql to work together
import psycopg2 as pg
import pandas.io.sql as pd_sql
from psycopg2 import connect
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

#get distances between coordinates
from geopy.distance import geodesic
import numpy as np

from sklearn.externals import joblib


from Backyard_House_Helper_Functions import geo_data_amenity_count_func, geo_data_dist_func<jupyter_output>/Users/agar/opt/anaconda3/envs/metis/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.
  warnings.warn(msg, category=FutureWarning)<jupyter_text>4.0 Pull in AirBnB Data to look at reducing features for an mvp prediction<jupyter_code>#Connect to sql database
params = {
    'host': 'localhost',  # connecting to _local_ version of psql
    'user': 'agar',
    'dbname': 'properties',    # DB connecting to
    'port': 5432          # port
}

# make connection by unpacking params
connection = pg.connect(**params)
#Pull target and features from AirBnB sql table
#Get only superhost properties
#Get relevant amenities and location information
#Limit review scores to content-based on property and check-in ease
query_airbnb_edited = """
select cast(regexp_replace(price, '\$|\.00|,', '','g') as int) as price, al.reviews_per_month, al.accommodates, al.bedrooms, al.bathrooms, al.beds, al.availability_365, 
al.review_scores_checkin, al.review_scores_cleanliness, al.review_scores_communication, 
al.review_scores_location, al.review_scores_value, al.property_type, al."latitude", al."longitude", 
regexp_replace(al."neighbourhood", '\ |\-|\/,', '_','g') as "neighborhood",

(count(al."neighbourhood") over
(PARTITION BY al."neighbourhood", al."bedrooms"))
         AS neigh_br_competition,
         
(case when al.amenities like '%Dishwasher%' then 1 else 0 end) as "dishwasher",
(case when al.amenities like '%Garden or backyard%' then 1 else 0 end) as "yard",
(case when al.amenities like '%Patio or balcony%' then 1 else 0 end) as "patio",
(case when al.amenities like '%Self check-in%' then 1 else 0 end) as "check_in_self",
(case when al.amenities like '%parking%' then 1 else 0 end) as "parking",
(case when al.amenities like '%pool%' then 1 else 0 end) as "pool",
(case when al.amenities like '%Long term%' then 1 else 0 end) as "long_term",
(case when al.amenities like '%Baby%' then 1 else 0 end) as "baby",
(case when al.amenities like '%Child%' then 1 else 0 end) as "children",
(case when al.amenities like '%Family%' then 1 else 0 end) as "family",
(case when al.amenities like '%24-hour%' then 1 else 0 end) as "check_in_24hrs",
al."amenities",
(case when al.amenities like '%Washer%' then 1 else 0 end) as "washer_dryer",
(case when al.amenities like '%Air conditioning%' then 1 else 0 end) as "air_con",
(case when al.amenities like '%Heating%' then 1 else 0 end) as "heating",
(case when al.amenities like '%Pets%' then 1 else 0 end) as "pets",
(case when al.amenities like '%Lockbox%' then 1 else 0 end) as "lockbox"

from airbnb_full_listings al
where room_type = 'Entire home/apt' and al.host_is_superhost = 't' and cast(regexp_replace(price, '\$|\.00|,', '','g') as int)<1000 and cast(regexp_replace(price, '\$|\.00|,', '','g') as int)>20
"""
#Pull x target and y features into DataFrame from sql
airbnb_df = pd.read_sql_query(query_airbnb_edited , connection, index_col=None, coerce_float=True, params=None, parse_dates=None, chunksize=None)
airbnb_df.columns
#Look at head
airbnb_df.head(20)<jupyter_output><empty_output><jupyter_text>Best Model for comparison<jupyter_code>#Create target and feature DataFrames
#Create X target dataframe from 'price'
#Create y feature dataframe from all other columns

X = airbnb_df.iloc[:, 1:]

y = airbnb_df.iloc[:, 0]

#Create lists to store cross-val scores for each metric in a list
# perform train/test split, get dummies for string-type features
X_train, X_test, y_train, y_test = \
    train_test_split(pd.get_dummies(X), y, test_size=0.2, random_state=42)

# Create 5-fold Cross Validation
kf = KFold(n_splits=5, random_state=42, shuffle=True)
kf.get_n_splits(X_train)
rmse_list = []
mae_list = []
r2_list = []

#Run model on each kfold split and record scores 
for train_ind, val_ind in kf.split(X_train):
    
    #assign train and validate sets
    X_tr, y_tr = X_train.iloc[train_ind], y_train.iloc[train_ind]
    X_val, y_val = X_train.iloc[val_ind], y_train.iloc[val_ind]
    
    #fit model
    xgb_model = xgb.XGBRegressor()
    xgb_model.fit(X_tr, y_tr)
    
    # score fit model on validation data
    xgb_preds = xgb_model.predict(X_val)
    
    val_score = xgb_model.score(X_val, y_val)
    rmse = np.sqrt(metrics.mean_squared_error(y_val,xgb_preds))
    mae = metrics.mean_absolute_error(y_val, xgb_preds)
    
    r2_list.append(val_score)
    rmse_list.append(rmse)
    mae_list.append(mae)

print("rmse cv avg: ", mean(rmse_list))
print("mae cv avg: ", mean(mae_list))
print("val cv avg: ", mean(r2_list))
pd.set_option('display.max_column', 10)
X_tr.transpose().head(100)
X.columns
# Save the model as a pickle in a file 
joblib.dump(xgb_model, 'xgb_model_sun.pkl') 
print(mae_list)<jupyter_output>[51.933369971806115, 52.160579128164, 52.00801329152912, 52.853233766085104, 49.19657202208685]<jupyter_text>Look at model performance with minimum features<jupyter_code>simplified_airbnb_df = airbnb_df.loc[:,["price", 'bedrooms', 'bathrooms', 
       'latitude', 'longitude']]

simplified_airbnb_df_2 = airbnb_df.loc[:,["price", 'accommodates', 'bedrooms', 'bathrooms', 
       'latitude', 'longitude']]
airbnb_df.loc[:,["price", 'bedrooms', 'bathrooms', 
       'latitude', 'longitude']]
#Create target and feature DataFrames
#Create X target dataframe from 'price'
#Create y feature dataframe from all other columns

X = simplified_airbnb_df.iloc[:, 1:]

y = simplified_airbnb_df.iloc[:, 0]

#Create lists to store cross-val scores for each metric in a list
# perform train/test split, get dummies for string-type features
X_train, X_test, y_train, y_test = \
    train_test_split(pd.get_dummies(X), y, test_size=0.2, random_state=42)

# Create 5-fold Cross Validation
kf = KFold(n_splits=5, random_state=42, shuffle=True)
kf.get_n_splits(X_train)
rmse_list = []
mae_list = []
r2_list = []

#Run model on each kfold split and record scores 
for train_ind, val_ind in kf.split(X_train):
    
    #assign train and validate sets
    X_tr, y_tr = X_train.iloc[train_ind], y_train.iloc[train_ind]
    X_val, y_val = X_train.iloc[val_ind], y_train.iloc[val_ind]
    
    #fit model
    xgb_model = xgb.XGBRegressor()
    xgb_model.fit(X_tr, y_tr)
    
    # score fit model on validation data
    xgb_preds = xgb_model.predict(X_val)
    
    val_score = xgb_model.score(X_val, y_val)
    rmse = np.sqrt(metrics.mean_squared_error(y_val,xgb_preds))
    mae = metrics.mean_absolute_error(y_val, xgb_preds)
    
    r2_list.append(val_score)
    rmse_list.append(rmse)
    mae_list.append(mae)

print("rmse cv avg: ", mean(rmse_list))
print("mae cv avg: ", mean(mae_list))
print("val cv avg: ", mean(r2_list))
#Create target and feature DataFrames
#Create X target dataframe from 'price'
#Create y feature dataframe from all other columns

X = simplified_airbnb_df_2.iloc[:, 1:]

y = simplified_airbnb_df_2.iloc[:, 0]

#Create lists to store cross-val scores for each metric in a list
# perform train/test split, get dummies for string-type features
X_train, X_test, y_train, y_test = \
    train_test_split(pd.get_dummies(X), y, test_size=0.2, random_state=42)

# Create 5-fold Cross Validation
kf = KFold(n_splits=5, random_state=42, shuffle=True)
kf.get_n_splits(X_train)
rmse_list = []
mae_list = []
r2_list = []

#Run model on each kfold split and record scores 
for train_ind, val_ind in kf.split(X_train):
    
    #assign train and validate sets
    X_tr, y_tr = X_train.iloc[train_ind], y_train.iloc[train_ind]
    X_val, y_val = X_train.iloc[val_ind], y_train.iloc[val_ind]
    
    #fit model
    xgb_model = xgb.XGBRegressor()
    xgb_model.fit(X_tr, y_tr)
    
    # score fit model on validation data
    xgb_preds = xgb_model.predict(X_val)
    
    val_score = xgb_model.score(X_val, y_val)
    rmse = np.sqrt(metrics.mean_squared_error(y_val,xgb_preds))
    mae = metrics.mean_absolute_error(y_val, xgb_preds)
    
    r2_list.append(val_score)
    rmse_list.append(rmse)
    mae_list.append(mae)

print("rmse cv avg: ", mean(rmse_list))
print("mae cv avg: ", mean(mae_list))
print("val cv avg: ", mean(r2_list))
#apply model to test set, get scores

xgb_preds = xgb_model.predict(X_test)
    
val_score = xgb_model.score(X_test, y_test)
rmse = np.sqrt(metrics.mean_squared_error(y_test,xgb_preds))
mae = metrics.mean_absolute_error(y_test, xgb_preds)

print("rmse : ", rmse)
print("mae : ", mae)
print("val : ", val_score)<jupyter_output>rmse :  90.48505242308774
mae :  55.54708293921743
val :  0.602346461768167<jupyter_text>4.2 Run AirBnB price prediction model on Backyard House Lot Data  Look at Backyard House Lot Data<jupyter_code>house_2 = pd.read_csv("adu_sun_calc_df.csv")
house_2.head()
house_2.columns<jupyter_output><empty_output><jupyter_text>Edit Backyard House Dataset to MVP features for modelling<jupyter_code>simplify_house_df = house_2.loc[:,['SitusFullAddress', 'adu_br', 'CENTER_LAT', 'CENTER_LON']]
simplify_house_df.head()<jupyter_output><empty_output><jupyter_text>Rename Features<jupyter_code>simplify_house_df.columns = ['address', 'bedrooms', 'latitude', 'longitude']<jupyter_output><empty_output><jupyter_text>Create Bathrooms feature for modeling and web app<jupyter_code>def bathroom_count(bedrooms):
    """ (float, float) -> int

    Return 2, or 1 indicating the number of bathrooms a backyard house could have
    based on the number of bedrooms.
    """
    if bedrooms == 3:
        bathrooms = 2
    else:
        bathrooms = 1
    return bathrooms
#apply bathroom function to Lot DataFrame
simplify_house_df["bathrooms"] =  simplify_house_df.apply(lambda x: bathroom_count(x.bedrooms), axis=1)<jupyter_output><empty_output><jupyter_text>Create area Feature for web app<jupyter_code>def house_area(bedrooms):
    """ (int) -> int

    Return the typical backyard house area for a 3 br
    is 1200 sf, 2 br is 700 sf, 1 br is 400 sf.

    """
    if bedrooms == 3:
        house_area = 1200
        return house_area
    elif bedrooms == 2:
        house_area = 700
        return house_area
    else:
        house_area = 400
    return house_area
#apply house area function to Lot DataFrame
simplify_house_df["house_area"] =  simplify_house_df.apply(lambda x: house_area(x.bedrooms), axis=1)
simplify_house_df.columns<jupyter_output><empty_output><jupyter_text>Create X matrix for predicting price<jupyter_code>X_df = simplify_house_df[['bedrooms', 'bathrooms', 
       'latitude', 'longitude']]
X_df.head()<jupyter_output><empty_output><jupyter_text>Apply price prediction model to test set<jupyter_code>xgb_preds = xgb_model.predict(X_df)
    
simplify_house_df["predicted_income"] = xgb_preds
simplify_house_df.head()<jupyter_output><empty_output><jupyter_text>Examine Results<jupyter_code>simplify_house_df.describe()
simplify_house_df.to_csv("backyard_house_predicted_income.csv")<jupyter_output><empty_output><jupyter_text>Round predicted income and classify income segment based on distribution<jupyter_code>rounded_income = simplify_house_df.round({"predicted_income":0}) 
rounded_income["predicted_income"] = rounded_income.predicted_income.astype(int)
rounded_income.head()
rounded_income.predicted_income.quantile(0.25)
rounded_income.predicted_income.quantile(0.75)
def income_category(predicted_income):
    """ (int) -> string

    Return low, medium, or high for given income predictions.

    """
    if predicted_income >= 155:
        income_segment = "high"
        return income_segment
    elif predicted_income >= 100:
        income_segment = "medium"
        return income_segment
    else:
        income_segment = "low"
    return income_segment
#apply income_segment function to Lot DataFrame
rounded_income["income_segment"] =  rounded_income.apply(lambda x: income_category(x.predicted_income), axis=1)

rounded_income.to_csv("backyard_house_income_segmentation.csv")
df = pd.read_csv("backyard_house_income_segmentation.csv")

df.head()<jupyter_output><empty_output><jupyter_text>prep look-up function for web app<jupyter_code>#get user address input
user_input = "1070 CASIANO RD LOS ANGELES CA 90049"

#mask dataframe row to user_input address
mask = df["address"] == user_input

#create new dataframe from mask
info = df[mask]

if info.shape[0] == 0:
    print("not good addy")
    
#create list from masked dataframe
prop_info = list(info.iloc[0])

#extract user output from list
#bedrooms
br = prop_info[2]

#baths
ba = prop_info[5]

#square feet
sf = prop_info[6]

#nightly price/income
price = prop_info[7]

#LA segment
segment = prop_info[8]

#create tuple to use in string
test_tuple = (price, segment, sf, br, ba)

#print output
print ("""Your nightly income prediction is $%d which is in the %s income segment for Los Angeles.\n
Your property can accomodate a %d square foot backyard house. \n
This is typically a %d bedroom , %d bath unit. \n""" % test_tuple
      )<jupyter_output>Your nightly income prediction is $332 which is in the high income segment for Los Angeles.

Your property can accomodate a 1200 square foot backyard house. 

This is typically a 3 bedroom , 2 bath unit.<jupyter_text>a quick look at the distribution of nightly incomes in this iteration<jupyter_code>import math

data = df["predicted_income"]
w = 20
n = math.ceil((data.max() - data.min())/w)
ax = plt.hist(data, bins = n)
# editing out the right tail
n = math.ceil((data.max() - data.min())/w)
ax = plt.hist(data, range = (data.min(), 400), bins = n)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10627049454594124, 'fraction_numerical': 0.02282004952151509, 'mean_word_length': 5.037979797979798, 'pattern_counts': {'":': 1, '<': 59, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '655329', 'license': 'MIT', 'n_tokens_mistral': 5624, 'n_tokens_neox': 5266, 'n_words': 1414, 'path': '04_adg_Nightly Price Predictions for Backyard Houses.ipynb', 'repo_name': 'Pamaland1/Prospecting_LA_Backyard_House_Income', 'size': '14950'}