<jupyter_start><jupyter_text>1.Business UnderstandingAirbnb operates an online marketplce for lodging, primarily homestays for vacation rentals, and tourism activities. This blog is an effort to interpret the Boston Airbnb data from kaggle and answer several questions below:- Build a model to predict property price- what features that influence the property price? - What time of year has the highest rental price?<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from sklearn.linear_model import LinearRegression<jupyter_output><empty_output><jupyter_text>2.Data Understanding 2.1.Data Gathering<jupyter_code>listing_df = pd.read_csv('listings.csv')
calendar_df = pd.read_csv('calendar.csv')
review_df = pd.read_csv('reviews.csv')
for data in [listing_df, calendar_df, review_df]:
    print(data.shape)
for data in [listing_df, calendar_df, review_df]:
    display(data.head())<jupyter_output><empty_output><jupyter_text>- listing data<jupyter_code>#check the datatype for each column and non-null values 
listing_df.info()
#remove the column which has all nan values since it cannot be filled with any reference value
listing_df.dropna(axis=1, how='all', inplace=True)
listing_df.shape<jupyter_output><empty_output><jupyter_text>- calendar data<jupyter_code>#check the datatype for each column and non-null values
calendar_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1308890 entries, 0 to 1308889
Data columns (total 4 columns):
 #   Column      Non-Null Count    Dtype 
---  ------      --------------    ----- 
 0   listing_id  1308890 non-null  int64 
 1   date        1308890 non-null  object
 2   available   1308890 non-null  object
 3   price       643037 non-null   object
dtypes: int64(1), object(3)
memory usage: 39.9+ MB<jupyter_text>- review data<jupyter_code>#check datatype
review_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 68275 entries, 0 to 68274
Data columns (total 6 columns):
 #   Column         Non-Null Count  Dtype 
---  ------         --------------  ----- 
 0   listing_id     68275 non-null  int64 
 1   id             68275 non-null  int64 
 2   date           68275 non-null  object
 3   reviewer_id    68275 non-null  int64 
 4   reviewer_name  68275 non-null  object
 5   comments       68222 non-null  object
dtypes: int64(3), object(3)
memory usage: 3.1+ MB<jupyter_text>2.2.Data Cleaning  Cleaning for listing dataset<jupyter_code>#drop na rows in bathrooms, bedrooms, beds 
listing_df.dropna(axis = 0, subset = ['bathrooms', 'bedrooms', 'beds'], inplace = True)

#creating new df with bathrooms, bedrooms, beds
df_rooms = listing_df[['bathrooms', 'bedrooms', 'beds']]

#converting the data type from string to integer
for room in df_rooms:
    df_rooms[room].astype(int)
    

    
listing_df[['bathrooms', 'bedrooms', 'beds']] = df_rooms[['bathrooms', 'bedrooms', 'beds']]
#replace $ from the price values and convert str into float data type
for fee in ['cleaning_fee', 'security_deposit', 'price']:
    listing_df[fee] = listing_df[fee].map(lambda x: x.replace("$", '').replace(",",''), na_action = 'ignore')
    listing_df[fee] = listing_df[fee].astype(float)
    listing_df[fee].fillna(listing_df[fee].median(), inplace = True)
#fill na for review columns, since the review data distribution show that the data
# is highly skewed to the left, choose mean to fillna 
for feature in['review_scores_rating', 'review_scores_accuracy','review_scores_cleanliness',
              'review_scores_checkin', 'review_scores_communication', 'review_scores_location',
              'review_scores_value', 'reviews_per_month']:
    listing_df[feature].fillna(listing_df[feature].mean(),inplace = True)
#for categorical values in the df

#create new df only has numerical columns
df_int = listing_df.select_dtypes(include = ['int64', 'float64'])

#add categorical columns to df_int
df_int[['room_type', 'neighbourhood_cleansed', 'cancellation_policy',
       'property_type']] = listing_df[['room_type', 'neighbourhood_cleansed', 'cancellation_policy','property_type']]
#drop the columns that have large number of null
df_int.drop(labels = ['square_feet'], axis = 1, inplace = True)
df_int.info()
#create dummy variables 
cleanup_nums = {"room_type": {"Entire home/apt": 1, "Private room": 2, "Shared room": 3},
                "cancellation_policy": {"moderate": 1, "flexible": 2, "strict": 3, "super_strict_30": 4}}      
df_int_copy = df_int
df_int_copy = df_int_copy.replace(cleanup_nums)

df_int_dummies = pd.get_dummies(df_int_copy)<jupyter_output><empty_output><jupyter_text>Cleaning for calendar dataset<jupyter_code>#extract month from dates columns
calendar_df['month'] = calendar_df.date.apply(lambda x: x.split('-')[1])
calendar_df['month'] = calendar_df['month'].replace({
    '01': 'Jan',
    '02': 'Feb',
    '03': 'Mar',
    '04': 'Apr',
    '05': 'May',
    '06': 'Jun',
    '07': 'Jul',
    '08': 'Aug',
    '09': 'Sep',
    '10': 'Oct',
    '11': 'Nov',
    '12': 'Dec'
})
#remove special character from price column
#convert to float
calendar_df.dropna(axis = 0, subset = ["price"], inplace = True)
calendar_df['price'] = calendar_df['price'].map(lambda x: x.replace("$", '').replace(",", ''), na_action = 'ignore').astype(float)
calendar_df.head()<jupyter_output><empty_output><jupyter_text>3.Modeling and Evaluation<jupyter_code>X = df_int_dummies.drop(labels = ["price"], axis = 1)
y = df_int_dummies['price']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.30, random_state = 42)

#create a function to fit the model and evaluate
def model_evaluate(model, X_train, y_train, X_test, y_test):
    '''
    take training to fit the model and test on testing data set
    '''
    model.fit(X_train, y_train)
    pred = model.predict(X_test)
    mae = mean_absolute_error(y_test, pred)
    print("MAE from {0}:{1}".format(model, mae))
    fig = plt.figure(figsize = (12, 8), dpi = 80)
    ax1 = fig.add_subplot(111)
    lines = plt.plot(y_test - pred, marker= 'o', linestyle = '')
    ax1.set_xlabel('True Value_{}'.format(model))
    ax1.set_ylabel('Predictions_{}'.format(model))
#random forest regressor
model_rf = RandomForestRegressor(n_estimators = 100, random_state = 0)
model_evaluate(model_rf, X_train, y_train, X_test, y_test)
model_rf.predict
#linear regression
model_lr = LinearRegression()
model_evaluate(model_lr, X_train, y_train, X_test, y_test)<jupyter_output>MAE from LinearRegression():171.42041198501872<jupyter_text>Based on the results from above two models, Random Forest Regreesor has a MAE of 47.2 comparing to Linear Regression MAe value 32596, which means Random Forest model is performing better.  4. Visualization - For the first question about the features that influence the property price.<jupyter_code>#count the total number of entries for each type in room_type feature
df_int.groupby(['room_type']).count()
#visualize the room_type and their mean price for each type
fig, axs = plt.subplots(1, 2, figsize = (13, 5), dpi = 80)
sns.countplot(x = 'room_type', data = df_int, palette = 'cool', ax = axs[0]).set_title("number of properties based on room type")
axs[0].set_xlabel('room type')

#visualize the distribution of pricing among each room type
sns.distplot(df_int[df_int.room_type == 'Private room']['price'], kde = False, ax = axs[1], label = 'Private room')
sns.distplot(df_int[df_int.room_type == 'Shared room']['price'], kde = False, ax = axs[1], label = 'Shared room')
sns.distplot(df_int[df_int.room_type == 'Entire home/apt']['price'], kde = False, ax = axs[1], label = 'Entire home/apt')

axs[1].set_xlim(0, 600)
axs[1].set_title('Room Type')
axs[1].set_xlabel('Price($)')
axs[1].legend()
#visualize the data btw selectd features and price column
fig, axs = plt.subplots(3, 2, figsize = (25, 30), dpi = 80)

df_roomtype = df_int.groupby(['room_type', 'bedrooms'], as_index = False).mean()

#neighbourhoods
neighbourhoods = df_int.groupby('neighbourhood_cleansed').agg(
    {
        'price': np.mean
    }
).reset_index()
sns.barplot(
    x='price', y='neighbourhood_cleansed',
    data=neighbourhoods.sort_values('price'),
    orient='h',
    palette='muted',
    ax=axs[0, 0]
)
axs[0, 0].set_title('Neighbourhood and Pricing')
axs[0, 0].set_xlabel('Mean Price ($)')
axs[0, 0].set_ylabel('')

#roomtype
sns.barplot(x = 'room_type', y = 'price', hue = 'bedrooms', data = df_roomtype, 
           palette = 'muted', ax = axs[0, 1])
axs[0, 1].set_title('Price difference in each room type based on bedrooms')
axs[0, 1].set_xlabel('Room Type')
axs[0, 1].legend()

#cancellation Policy
sns.boxplot(x='price', y='cancellation_policy', fliersize=1, linewidth=0.75,
            data=df_int, palette='muted', ax=axs[1, 0],
            order=['flexible', 'moderate', 'strict', 'super_strict_30'])
axs[1, 0].set_xlim(0, 600)
axs[1, 0].set_title('Cancellation Policy and Pricing')
axs[1, 0].set_xlabel('Price ($)')
axs[1, 0].set_ylabel('')

# Property type
sns.boxplot(x='price', y='property_type', fliersize=1, linewidth=0.75,
            data=df_int, palette='muted', ax=axs[1, 1])
axs[1, 1].set_xlim(0, 600)
axs[1, 1].set_title('Property Type and Pricing')
axs[1, 1].set_xlabel('Price ($)')
axs[1, 1].set_ylabel('')

#calendar
cats = ['Jan', 'Feb', 'Mar', 'Apr','May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
calendar_df['month'] = pd.Categorical(calendar_df['month'], ordered=True, categories=cats)

plt.figure(figsize=(10,5))

sns.lineplot(data = calendar_df, x = "month", y = "price", sort=True, ax = axs[2,0])

axs[2, 0].set_title("Month and Price")
axs[2, 0].set_xlabel('Month')
axs[2, 0].set_ylabel('Price ($)')


plt.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14427257102416338, 'fraction_numerical': 0.031038317662521485, 'mean_word_length': 4.930455635491607, 'pattern_counts': {'":': 9, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '479177', 'license': 'MIT', 'n_tokens_mistral': 3726, 'n_tokens_neox': 3455, 'n_words': 898, 'path': 'Boston Aribnb Data.ipynb', 'repo_name': 'YiWang-Evonne/Boston-Airbnb-Data', 'size': '9891'}