<jupyter_start><jupyter_text>AirBnB Price Prediction<jupyter_code># import the library
%matplotlib inline

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# sklearn :: utils
from sklearn.model_selection import train_test_split

# sklearn :: models
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import Lasso


# sklearn :: evaluation metrics
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error

sns.set_style('whitegrid')<jupyter_output><empty_output><jupyter_text>Load the csv Data Files into Dataframe<jupyter_code>df_train = pd.read_csv('data/train.csv')
df_test = pd.read_csv('data/test.csv')
# df_zip = pd.read_csv('data/zipcode.csv')
print(df_train.shape, df_test.shape)<jupyter_output>(51000, 29) (23111, 28)<jupyter_text>View Data Features, Data Type and Missing Values<jupyter_code>print(df_train.columns)
df_train.head()
# Data types of the Feature
df_train.dtypes
# Find the missing values
print(df_train.isnull().sum())
print(df_train.isnull().sum())<jupyter_output>id                            0
log_price                     0
property_type                 0
room_type                     0
amenities                     0
accommodates                  0
bathrooms                   133
bed_type                      0
cancellation_policy           0
cleaning_fee                  0
city                          0
description                   0
first_review              10856
host_has_profile_pic        127
host_identity_verified      127
host_response_rate        12551
host_since                  127
instant_bookable              0
last_review               10826
latitude                      0
longitude                     0
name                          0
neighbourhood              4675
number_of_reviews             0
review_scores_rating      11450
thumbnail_url              5629
zipcode                     645
bedrooms                     63
beds                         85
dtype: int64<jupyter_text>Convert Data Type, Handle Missing Values & outliers and Fearure Re-engineering  Feature : property_type<jupyter_code>print(df_train.property_type.unique())
print(df_train['property_type'].value_counts())
df_train.groupby(by='property_type')['log_price'].mean()
# Find the difference between 'property_type' fearure in both dataframes
df1 = df_train.property_type.unique()
df2 = df_test.property_type.unique()
diff = list(set(df1) - set(df2))
print(diff)

# Drop the rows from df_train which contains ['property_type'] 'Cave', 'Island', 'Earth House', 'Lighthouse'
for i in diff:
    df_train = df_train[df_train['property_type'] != i]

# Create new features from property_type
df_property_type = pd.get_dummies(df_train['property_type']).astype(int)
df_train = pd.concat([df_train, df_property_type], axis=1)

# Delete 'property_type' column from df_train
del df_train['property_type']
df_train.head()
# Create new features from property_type
df_property_type = pd.get_dummies(df_test['property_type'])
df_test = pd.concat([df_test, df_property_type], axis=1)

# Delete 'property_type' column from df_train
del df_test['property_type']<jupyter_output><empty_output><jupyter_text>Feature : room_type<jupyter_code>print(df_train.room_type.unique())
print(df_train['room_type'].value_counts())
df_train.groupby(by='room_type')['log_price'].mean()
# Find the difference between 'room_type' fearure in both dataframes
df1 = df_train.room_type.unique()
df2 = df_test.room_type.unique()
diff = list(set(df1) - set(df2))
print(diff)

# Create new features from room_type
df_room_type = pd.get_dummies(df_train['room_type']).astype(int)
df_train = pd.concat([df_train, df_room_type], axis=1)

# Delete 'room_type' column from df_train
del df_train['room_type']
# Create new features from room_type
df_room_type = pd.get_dummies(df_test['room_type'])
df_test = pd.concat([df_test, df_room_type], axis=1)

# Delete 'property_type' column from df_train
del df_test['room_type']<jupyter_output><empty_output><jupyter_text>Feature: amenities<jupyter_code># Create new feature with total number of amenities
df_train['#amenities'] = df_train['amenities'].str.count(",") + 1

df_train['family'] = np.where(df_train['amenities'].str.contains("Family")==True, '1', '0')
df_train['family'] = df_train['family'].astype(int)

df_train['pets'] = np.where(df_train['amenities'].str.contains("Pets allowed")==True, '1', '0')
df_train['pets'] = df_train['pets'].astype(int)

df_train['event'] = np.where(df_train['amenities'].str.contains("Suitable for events")==True, '1', '0')
df_train['event'] = df_train['event'].astype(int)

df_train['kitchen'] = np.where(df_train['amenities'].str.contains("Kitchen")==True, '1', '0')
df_train['kitchen'] = df_train['kitchen'].astype(int)

df_train['heating'] = np.where(df_train['amenities'].str.contains("Heating")==True, '1', '0')
df_train['heating'] = df_train['heating'].astype(int)

df_train['gym'] = np.where(df_train['amenities'].str.contains("Gym")==True, '1', '0')
df_train['gym'] = df_train['gym'].astype(int)

df_train['elevator'] = np.where(df_train['amenities'].str.contains("Elevator")==True, '1', '0')
df_train['elevator'] = df_train['elevator'].astype(int)

df_train['pool'] = np.where(df_train['amenities'].str.contains("Pool")==True, '1', '0')
df_train['pool'] = df_train['pool'].astype(int)

# Delete 'amenities' column from df_train
del df_train['amenities']
# Create new features from amenities
df_test['#amenities'] = df_test['amenities'].str.count(",") + 1

df_test['family'] = np.where(df_test['amenities'].str.contains("Family")==True, '1', '0')
df_test['family'] = df_test['family'].astype(int)

df_test['pets'] = np.where(df_test['amenities'].str.contains("Pets allowed")==True, '1', '0')
df_test['pets'] = df_test['pets'].astype(int)

df_test['event'] = np.where(df_test['amenities'].str.contains("Suitable for events")==True, '1', '0')
df_test['event'] = df_test['event'].astype(int)

df_test['kitchen'] = np.where(df_test['amenities'].str.contains("Kitchen")==True, '1', '0')
df_test['kitchen'] = df_test['kitchen'].astype(int)

df_test['heating'] = np.where(df_test['amenities'].str.contains("Heating")==True, '1', '0')
df_test['heating'] = df_test['heating'].astype(int)

df_test['gym'] = np.where(df_test['amenities'].str.contains("Gym")==True, '1', '0')
df_test['gym'] = df_test['gym'].astype(int)

df_test['elevator'] = np.where(df_test['amenities'].str.contains("Elevator")==True, '1', '0')
df_test['elevator'] = df_test['elevator'].astype(int)

df_test['pool'] = np.where(df_test['amenities'].str.contains("Pool")==True, '1', '0')
df_test['pool'] = df_test['pool'].astype(int)

# Delete 'amenities' column from df_train
del df_test['amenities']<jupyter_output><empty_output><jupyter_text>Feature: accommodates<jupyter_code>print(df_train['accommodates'].value_counts())
df_train.groupby(by='accommodates')['log_price'].mean()
# Find the difference between 'accommodates' fearure in both dataframes
df1 = df_train.accommodates.unique()
df2 = df_test.accommodates.unique()
diff = list(set(df1) - set(df2))
print(diff)<jupyter_output>[]<jupyter_text>Feature: bathrooms<jupyter_code>print(df_train['bathrooms'].value_counts())
# Find the difference between 'bathrooms' fearure in both dataframes
df1 = df_train.bathrooms.unique()
df2 = df_test.bathrooms.unique()
diff = list(set(df1) - set(df2))
print(diff)

# Find the missing values
print(df_train['bathrooms'].isnull().sum())

# Replace NaN with Mean value in bathroom feature / column
df_train['bathrooms'].fillna(round(df_train['bathrooms'].mean()), inplace=True)<jupyter_output>[nan]
133<jupyter_text>Feature: bed_type<jupyter_code>print(df_train['bed_type'].value_counts())
# Find the difference between 'bed_type' fearure in both dataframes
df1 = df_train.bed_type.unique()
df2 = df_test.bed_type.unique()
diff = list(set(df1) - set(df2))
print(diff)

# Create new features from bed_type
df_bed_type = pd.get_dummies(df_train['bed_type']).astype(int)
df_train = pd.concat([df_train, df_bed_type], axis=1)

# Delete 'bed_type' column from df_train
del df_train['bed_type']
# Create new features from bed_type
df_bed_type = pd.get_dummies(df_test['bed_type'])
df_test = pd.concat([df_test, df_bed_type], axis=1)

# Delete 'bed_type' column from df_train
del df_test['bed_type']<jupyter_output><empty_output><jupyter_text>Feature: cancellation_policy<jupyter_code>print(df_train['cancellation_policy'].value_counts())
# Find the difference between 'cancellation_policy' fearure in both dataframes
df1 = df_train.cancellation_policy.unique()
df2 = df_test.cancellation_policy.unique()
diff = list(set(df1) - set(df2))
print(diff)

# Create new features from cancellation_policy
df_cancellation_policy = pd.get_dummies(df_train['cancellation_policy']).astype(int)
df_train = pd.concat([df_train, df_cancellation_policy], axis=1)

# Delete 'cancellation_policy' column from df_train
del df_train['cancellation_policy']
df_cancellation_policy = pd.get_dummies(df_test['cancellation_policy']).astype(int)
df_test = pd.concat([df_test, df_cancellation_policy], axis=1)

# Delete 'cancellation_policy' column from df_train
del df_test['cancellation_policy']<jupyter_output><empty_output><jupyter_text>Feature: city<jupyter_code>print(df_train['city'].value_counts())
# Find the difference between 'city' fearure in both dataframes
df1 = df_train.city.unique()
df2 = df_test.city.unique()
diff = list(set(df1) - set(df2))
print(diff)

# Create new features from city
df_city = pd.get_dummies(df_train['city']).astype(int)
df_train = pd.concat([df_train, df_city], axis=1)

# # Delete 'city' column from df_train
# del df_train['city']
# df_train
# Create new features from city
df_city = pd.get_dummies(df_test['city'])
df_test = pd.concat([df_test, df_city], axis=1)<jupyter_output><empty_output><jupyter_text>Feature: description<jupyter_code># Delete column from df_train
del df_train['description']
# Delete column from df_train
del df_test['description']<jupyter_output><empty_output><jupyter_text>Feature: first_review<jupyter_code># Transform object/string Date/Time data to datetime
df_train['first_review'] = pd.to_datetime(df_train['first_review'])

# Delete column from df_train
del df_train['first_review']
# Delete column from df_train
del df_test['first_review']<jupyter_output><empty_output><jupyter_text>Feature: host_has_profile_pic<jupyter_code># Replace t with 1, f with 0 and NaN with 0 of host_identity_verified feature
df_train['host_has_profile_pic'].mask(df_train['host_has_profile_pic'] == "t", "1", inplace=True)
df_train['host_has_profile_pic'].mask(df_train['host_has_profile_pic'] == "f", "0", inplace=True)
df_train['host_has_profile_pic'].fillna(0.0, inplace=True)

# Convert Data Type to Float
df_train['host_has_profile_pic'] = df_train['host_has_profile_pic'].astype(float)
# Replace t with 1, f with 0 and NaN with 0 of host_identity_verified feature
df_test['host_has_profile_pic'].mask(df_test['host_has_profile_pic'] == "t", "1", inplace=True)
df_test['host_has_profile_pic'].mask(df_test['host_has_profile_pic'] == "f", "0", inplace=True)
df_test['host_has_profile_pic'].fillna(0.0, inplace=True)

# Convert Data Type to Float
df_test['host_has_profile_pic'] = df_test['host_has_profile_pic'].astype(float)<jupyter_output><empty_output><jupyter_text>Feature: host_identity_verified<jupyter_code># Replace t with 1, f with 0 and NaN with 0 of host_identity_verified feature
df_train['host_identity_verified'].mask(df_train['host_identity_verified'] == "t", "1", inplace=True)
df_train['host_identity_verified'].mask(df_train['host_identity_verified'] == "f", "0", inplace=True)
df_train['host_identity_verified'].fillna(0.0, inplace=True)

# Convert Data Type to Float
df_train['host_identity_verified'] = df_train['host_identity_verified'].astype(float)
# Replace t with 1, f with 0 and NaN with 0 of host_identity_verified feature
df_test['host_identity_verified'].mask(df_test['host_identity_verified'] == "t", "1", inplace=True)
df_test['host_identity_verified'].mask(df_test['host_identity_verified'] == "f", "0", inplace=True)
df_test['host_identity_verified'].fillna(0.0, inplace=True)

# Convert Data Type to Float
df_test['host_identity_verified'] = df_test['host_identity_verified'].astype(float)<jupyter_output><empty_output><jupyter_text>Feature: host_response_rate<jupyter_code># Find the missing values
print(df_train['host_response_rate'].isnull().sum())
# Delete % sign from host_response_rate data and convert the data from object to integer
df_train['host_response_rate'] = df_train['host_response_rate'].str.replace('%', '')
df_train['host_response_rate'].fillna(0, inplace=True)

# Convert data type to Integer
df_train['host_response_rate'] = df_train['host_response_rate'].astype(int)

# Mean of host_response_rate without considering 0 values 
mean_host_response_rate = round(df_train['host_response_rate'].mean(skipna=True))

# Replace 0 with Mean value
df_train['host_response_rate'].mask(df_train['host_response_rate'] == 0, mean_host_response_rate, inplace=True)
# Delete % sign from host_response_rate data and convert the data from object to integer
df_test['host_response_rate'] = df_test['host_response_rate'].str.replace('%', '')
df_test['host_response_rate'].fillna(0, inplace=True)

# Convert data type to Integer
df_test['host_response_rate'] = df_test['host_response_rate'].astype(int)

# Mean of host_response_rate without considering 0 values 
mean_host_response_rate = round(df_test['host_response_rate'].mean(skipna=True))

# Replace 0 with Mean value
df_test['host_response_rate'].mask(df_test['host_response_rate'] == 0, mean_host_response_rate, inplace=True)<jupyter_output><empty_output><jupyter_text>Feature: host_since<jupyter_code># Transform object/string Date/Time data to datetime
df_train['host_since'] = pd.to_datetime(df_train['host_since'])

df_train['host_since_year'] = df_train['host_since'].dt.year
print(round(df_train['host_since_year'].mean(skipna=True)))
df_train['host_since_year'].fillna(round(df_train['host_since_year'].mean()), inplace=True)

# Delete column from df_train
del df_train['host_since']
# Transform object/string Date/Time data to datetime
df_test['host_since'] = pd.to_datetime(df_test['host_since'])

df_test['host_since_year'] = df_test['host_since'].dt.year
print(round(df_test['host_since_year'].mean(skipna=True)))
df_test['host_since_year'].fillna(round(df_test['host_since_year'].mean()), inplace=True)

# Delete column from df_train
del df_test['host_since']<jupyter_output>2014<jupyter_text>Feature: instant_bookable<jupyter_code>print(df_train['instant_bookable'].value_counts())
# Replace t with 1, f with 0 and NaN with 0 of host_identity_verified feature
df_train['instant_bookable'].mask(df_train['instant_bookable'] == "t", "1", inplace=True)
df_train['instant_bookable'].mask(df_train['instant_bookable'] == "f", "0", inplace=True)

# Convert Data Type to Integer
df_train['instant_bookable'] = df_train['instant_bookable'].astype(int)
# Replace t with 1, f with 0 and NaN with 0 of host_identity_verified feature
df_test['instant_bookable'].mask(df_test['instant_bookable'] == "t", "1", inplace=True)
df_test['instant_bookable'].mask(df_test['instant_bookable'] == "f", "0", inplace=True)

# Convert Data Type to Integer
df_test['instant_bookable'] = df_test['instant_bookable'].astype(int)<jupyter_output><empty_output><jupyter_text>Feature: last_review<jupyter_code># Transform object/string Date/Time data to datetime
df_train['last_review'] = pd.to_datetime(df_train['last_review'])

# Create new Feature with Year
df_train['last_review_year'] = df_train['last_review'].dt.year

# Replace NaN with 0 value
df_train['last_review_year'].fillna(0, inplace=True)

# Create new features from neighbourhood
df_last_review_year = pd.get_dummies(df_train['last_review_year']).astype(int)
df_train = pd.concat([df_train, df_last_review_year], axis=1)

# Delete column from df_train
del df_train['last_review']
# Transform object/string Date/Time data to datetime
df_test['last_review'] = pd.to_datetime(df_test['last_review'])

# Create new Feature with Year
df_test['last_review_year'] = df_test['last_review'].dt.year

# Replace NaN with 0 value
df_test['last_review_year'].fillna(0, inplace=True)

# Create new features from neighbourhood
df_last_review_year = pd.get_dummies(df_test['last_review_year']).astype(int)
df_test = pd.concat([df_test, df_last_review_year], axis=1)

# Delete column from df_train
del df_test['last_review']<jupyter_output><empty_output><jupyter_text>Feature: 'name'<jupyter_code># Delete column from df_train
del df_train['name']
# Delete column from df_train
del df_test['name']
print(df_train.shape, df_test.shape)<jupyter_output>(50992, 88) (23111, 89)<jupyter_text>Feature: neighbourhood<jupyter_code>def remove_whitespace(x):
    try:
        # remove spaces inside and outside of string
        x = "".join(x.split())

    except:
        pass
    return x

# Apply remove_whitespace to column only
df_train['neighbourhood'] = df_train['neighbourhood'].apply(remove_whitespace)

# Apply to remove_whitespace to entire Dataframe
# df = df.applymap(remove_whitespace)
# print(df)
# Delete special charecter from column string
df_train['neighbourhood'] = df_train['neighbourhood'].str.replace('"', '')

mapping = dict(df_train.dropna()[['zipcode', 'neighbourhood']].values)
df_train['neighbourhood'].fillna(df_train['zipcode'].map(mapping), inplace=True)

# df_train.dropna(how='any', inplace=True)

df_train = df_train[pd.notnull(df_train['neighbourhood'])]

# Create new features from neighbourhood
df_neighbourhood = pd.get_dummies(df_train['neighbourhood']).astype(int)
df_train = pd.concat([df_train, df_neighbourhood], axis=1)

# # Delete column from df_train
# del df_train['neighbourhood']
# Delete special charecter from column string
df_test['neighbourhood'] = df_test['neighbourhood'].str.replace('"', '')

mapping = dict(df_test.dropna()[['zipcode', 'neighbourhood']].values)
df_test['neighbourhood'].fillna(df_test['zipcode'].map(mapping), inplace=True)

# Create new features from neighbourhood
df_neighbourhood = pd.get_dummies(df_test['neighbourhood']).astype(int)
df_test = pd.concat([df_test, df_neighbourhood], axis=1)

# # Delete column from df_train
# del df_test['neighbourhood']
df1 = df_train.neighbourhood.unique()
df2 = df_test.neighbourhood.unique()
diff = list(set(df1) - set(df2))
print(diff)

diff = list(set(df2)-set(df1))
print(diff)
print(df_train.shape, df_test.shape)
# df_train.columns = df_train.columns.to_series().apply(lambda x: x.strip())
# df_test.columns = df_test.columns.to_series().apply(lambda x: x.strip())

del df_train['neighbourhood']
del df_test['neighbourhood']<jupyter_output><empty_output><jupyter_text>Feature: review_scores_rating<jupyter_code># Replace NaN with No Reviews value in review_scores_rating feature / column
df_train['review_scores_rating'].fillna('No Reviews', inplace=True)

# convert review_scores_rating into buckets
def convert_scores_buckets(val):
    if val == 'No Reviews':
        return 'No Reviews'
    elif val >= 95:
        return '95-100'
    elif val >= 90 and val < 95:
        return '90-94'
    elif val >= 85 and val < 90:
        return '85-89'
    elif val >= 80 and val < 85:
        return '80-84'
    elif val >= 70 and val < 80:
        return '70-79'
    elif val >= 60 and val < 70:
        return '60-69'
    elif val >= 50 and val < 60:
        return '50-59'
    elif val >= 40 and val < 50:
        return '40-49'
    elif val >= 30 and val < 40:
        return '30-39'
    elif val >= 20 and val < 30:
        return '20-29'
    elif val >= 10 and val < 20:
        return '10-19'
    elif val < 10:
        return '0-9'

df_train['review_scores_rating'] = df_train['review_scores_rating'].apply(convert_scores_buckets)
print ('Unique Values in the Column:', np.unique(df_train['review_scores_rating']))
# print(df_train['review_scores_rating'].value_counts())

# Create new features from review_scores_rating
df_review_scores_buckets = pd.get_dummies(df_train['review_scores_rating']).astype(int)
df_train = pd.concat([df_train, df_review_scores_buckets], axis=1)

# Delete column from df_train
del df_train['review_scores_rating']
# Replace NaN with No Reviews value in review_scores_rating feature / column
df_test['review_scores_rating'].fillna('No Reviews', inplace=True)

# convert review_scores_rating into buckets
def convert_scores_buckets(val):
    if val == 'No Reviews':
        return 'No Reviews'
    elif val >= 95:
        return '95-100'
    elif val >= 90 and val < 95:
        return '90-94'
    elif val >= 85 and val < 90:
        return '85-89'
    elif val >= 80 and val < 85:
        return '80-84'
    elif val >= 70 and val < 80:
        return '70-79'
    elif val >= 60 and val < 70:
        return '60-69'
    elif val >= 50 and val < 60:
        return '50-59'
    elif val >= 40 and val < 50:
        return '40-49'
    elif val >= 30 and val < 40:
        return '30-39'
    elif val >= 20 and val < 30:
        return '20-29'
    elif val >= 10 and val < 20:
        return '10-19'
    elif val < 10:
        return '0-9'

df_test['review_scores_rating'] = df_test['review_scores_rating'].apply(convert_scores_buckets)
print ('Unique Values in the Column:', np.unique(df_test['review_scores_rating']))
# print(df_train['review_scores_rating'].value_counts())

# Create new features from review_scores_rating
df_review_scores_buckets = pd.get_dummies(df_test['review_scores_rating']).astype(int)
df_test = pd.concat([df_test, df_review_scores_buckets], axis=1)

# Delete column from df_train
del df_test['review_scores_rating']<jupyter_output>Unique Values in the Column: ['20-29' '30-39' '40-49' '50-59' '60-69' '70-79' '80-84' '85-89' '90-94'
 '95-100' 'No Reviews']<jupyter_text>Feature: thumbnail_url<jupyter_code># Delete column from df_train
del df_train['thumbnail_url']
# Delete column from df_train
del df_test['thumbnail_url']<jupyter_output><empty_output><jupyter_text>Feature: zipcode<jupyter_code>print(df_train['zipcode'].value_counts())
# Delete column from df_train
del df_train['zipcode']
# Delete column from df_train
del df_test['zipcode']<jupyter_output><empty_output><jupyter_text>Feature: bedrooms<jupyter_code># Replace NaN with Mean value in bedrooms feature / column
df_train['bedrooms'].fillna(round(df_train['bedrooms'].mean()), inplace=True)<jupyter_output><empty_output><jupyter_text>Feature: beds<jupyter_code># Replace NaN with Mean value in beds feature / column
df_train['beds'].fillna(round(df_train['beds'].mean()), inplace=True)<jupyter_output><empty_output><jupyter_text>Handle Outlier<jupyter_code>print('mean', np.mean(df_train['log_price']))
print('std', np.std(df_train['log_price']))
plt.hist(df_train['log_price'], bins=30)
plt.show()
print(df_train['accommodates'].value_counts())
print('mean', np.mean(df_train['accommodates']))
print('std', np.std(df_train['accommodates']))
df_train.groupby(by='accommodates')['log_price'].mean()
print(df_test['accommodates'].value_counts())
print(df_test['accommodates'].value_counts(normalize=True) * 100)
# df['gender'].value_counts(normalize=True) * 100
# Find the missing values
print(df_train.isnull().sum())
# # Correlation
# df_temp = df_train.filter(['log_price','accommodates', 'bathrooms', 'bedrooms', 'beds', 'Real Bed', 'Shared room', 
#                            'Entire home/apt', 'Private room', 'cleaning_fee', 'review_scores_rating', 
#                            'host_since_year', 'Boston', 'Chicago', 'DC', 'LA', 'NYC', 'SF', 'c_distance', 'Condominium', 'pets', 'event',
#                           'kitchen', 'heating', 'gym', 'elevator', 'pool', '#amenities'], axis=1)
# df_temp.corr()<jupyter_output><empty_output><jupyter_text>Add external feature in the dataset<jupyter_code>data = [['NYC', 40.71427, -74.00597], ['LA', 34.05223, -118.24368], ['SF', 37.77493, -122.41942], ['DC', 38.89511, -77.03637], 
        ['Chicago', 41.85003, -87.65005],['Boston', 42.35843, -71.05977]]
df_c = pd.DataFrame(data,columns=['City','c_lat', 'c_lon'])
df_c
df_merged = pd.merge(left=df_train, right=df_c, how='left', left_on=['city'], right_on=['City'])

df_train = df_merged
# df_train = df_train[(df_train['accommodates']<9)]
# df_train = df_train[df_train['property_type'].value_counts()>100]
import pyproj

geod = pyproj.Geod(ellps='WGS84') #Distance will be measured on this ellipsoid - more accurate than a spherical method

df_t = geod.inv(41.849684, -87.676270, 41.85003, -87.65005)
print(df_t)
# df_train['c_distance'] = geod.inv(df_train['latitude'], df_train['longitude'], df_train['c_lat'], df_train['c_lon'])

#Get distance between pairs of lat-lon points
def Distance(lat1,lon1,lat2,lon2):
  az12,az21,dist = geod.inv(lon1,lat1,lon2,lat2)
  return dist

#Add/update a column to the data frame with the distances (in metres)
df_train['c_distance'] = Distance(df_train['latitude'].tolist(),df_train['longitude'].tolist(),
                                  df_train['c_lat'].tolist(),df_train['c_lon'].tolist())
df_train.head(10)
# Delete column from df_train
del df_train['city']
del df_train['City']
# Add new feature c_distance

df_merged_t = pd.merge(left=df_test, right=df_c, how='left', left_on=['city'], right_on=['City'])
df_test = df_merged_t

#Get distance between pairs of lat-lon points
def Distance(lat1,lon1,lat2,lon2):
  az12,az21,dist = geod.inv(lon1,lat1,lon2,lat2)
  return dist

#Add/update a column to the data frame with the distances (in metres)
df_test['c_distance'] = Distance(df_test['latitude'].tolist(),df_test['longitude'].tolist(),
                                  df_test['c_lat'].tolist(),df_test['c_lon'].tolist())
data = [['NYC', 40.71427, -74.00597], ['LA', 34.05223, -118.24368], ['SF', 37.77493, -122.41942], ['DC', 38.89511, -77.03637], 
        ['Chicago', 41.85003, -87.65005],['Boston', 42.35843, -71.05977]]
df_c = pd.DataFrame(data,columns=['City','c_lat', 'c_lon'])
df_c
# Delete column from df_train
del df_test['city']
del df_test['City']
df_train.reset_index(drop=True, inplace=True)
df_test.reset_index(drop=True, inplace=True)<jupyter_output><empty_output><jupyter_text>Experiment<jupyter_code>feature = list(df_train)

# items to be removed 
unwanted_feature = {'id','log_price', 'latitude', 'longitude', 'c_lat', 'c_lon'}


list1 = [ele for ele in feature if ele not in unwanted_feature]

list1
# select the columns
X_columns = list1
y_column = ['log_price']
# split the data using sklearn

threshold = 0.8
X = df_train[X_columns]
y = df_train[y_column]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1.0-threshold, shuffle=True)

print('X_train', X_train.shape)
print('y_train', y_train.shape)
print('X_test', X_test.shape)
print('y_test', y_test.shape)
def model_training(model_name, model, X_train, y_train):
    model.fit(X_train, y_train)
    return model
    
def model_prediction(model, X_test):
    y_pred = model.predict(X_test)
    return y_pred

def model_evaluation(model_name, y_test, y_pred):
    print(model_name)
    print('MAE', mean_absolute_error(y_test, y_pred))
    print('RMSE', np.sqrt(mean_squared_error(y_test, y_pred)))
#     plt.scatter(y_test, y_pred, alpha=0.3)
#     plt.plot(range(0,5000000, 100), range(0,5000000, 100), '--r', alpha=0.3, label='Line1')
#     plt.title(model_name)
#     plt.xlabel('True Value')
#     plt.ylabel('Predict Value')
#     plt.xlim([0, 5000000])
#     plt.ylim([0, 5000000])
#     plt.show()
    print('')

def run_experiment(model_name, model, X_train, y_train, X_test):
    train_model = model_training(model_name, model, X_train, y_train)
    predictions = model_prediction(train_model, X_test)
    model_evaluation(model_name, y_test, predictions)
    
# run_experiment('Linear Regression', LinearRegression(), X_train, y_train, X_test)
# run_experiment('KNN 5', KNeighborsRegressor(5), X_train, y_train, X_test)
# run_experiment('KNN 2', KNeighborsRegressor(2), X_train, y_train, X_test)
# run_experiment('Decision Tree', DecisionTreeRegressor(), X_train, y_train, X_test)
# run_experiment('Random Forest 10', RandomForestRegressor(10), X_train, y_train, X_test)
# run_experiment('Random Forest 100', RandomForestRegressor(100), X_train, y_train, X_test)
# run_experiment('Random Forest 200', RandomForestRegressor(200), X_train, y_train, X_test)
# run_experiment('Gradient Boosting', GradientBoostingRegressor(), X_train, y_train, X_test)<jupyter_output><empty_output><jupyter_text>Model Training<jupyter_code># train a linear regression
model = GradientBoostingRegressor()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)<jupyter_output>C:\ProgramData\Anaconda3\lib\site-packages\sklearn\utils\validation.py:761: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)<jupyter_text>Model Evaluation<jupyter_code>rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print('RMSE', round(rmse, 4))
plt.scatter(y_test, y_pred, alpha=0.3)
plt.plot(range(0,10), range(0,10), '--r', alpha=0.3, label='Line1')
plt.title('Gradient Boosting')
plt.xlabel('True Value')
plt.ylabel('Predict Value')
plt.show()<jupyter_output>RMSE 0.4267<jupyter_text>Prepare submission<jupyter_code>df_prediction = df_test[X_columns].fillna(0.0)
df_test['log_price'] = model.predict(df_prediction)
df_test[['id', 'log_price']]
df_test[['id', 'log_price']].to_csv('Submission/AirBnB_Submission_5.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14411537297878776, 'fraction_numerical': 0.026759000907654554, 'mean_word_length': 6.187243295482, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '32', 'dir': 'jupyter-structured-clean-dedup', 'id': '601906', 'license': 'MIT', 'n_tokens_mistral': 11174, 'n_tokens_neox': 10516, 'n_words': 2043, 'path': 'AirBnB_Submission_5.ipynb', 'repo_name': 'bmshahrier/kaggle-airbnb', 'size': '29747'}