<jupyter_start><jupyter_text>Importing and Installing Libraries<jupyter_code>!pip install eli5
!pip install category_encoders
import pandas as pd
pd.set_option('display.max_columns', None)

import matplotlib.pyplot as plt
plt.style.use('seaborn')
import seaborn as sns

import functools

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline
from sklearn.metrics import r2_score

import category_encoders as ce

import eli5
from eli5.sklearn import PermutationImportance<jupyter_output>/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.metrics.scorer module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.metrics. Anything that cannot be imported from sklearn.metrics is now part of the private API.
  warnings.warn(message, FutureWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.feature_selection.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.feature_selection. Anything that cannot be imported from sklearn.feature_selection is now part of the private API.
  warnings.warn(message, FutureWarning)
Using TensorFlow backend.<jupyter_text>Loading and Reviewing Data Shape<jupyter_code># Loading up the original dataframe.

berlin = pd.read_csv('https://raw.githubusercontent.com/BuildWeekAirbnbOptimal1/Data-Science/master/Berlin.csv')
print(berlin.shape)
berlin.head()
# Checking the names of the columns in this dataframe.

berlin.columns<jupyter_output><empty_output><jupyter_text>Preprocessing  Deciding what columns to keep<jupyter_code># Creating the first df with dropping the columns which I don't think we will need at all.
# Dropped columns are those with mainly high cardinality.
# Keeping that columns which a new renter can normally directly influence.

columns_to_keep = ['id', 'space', 'description', 'host_response_time', 
                   'host_response_rate', 'host_has_profile_pic', 
                   'neighbourhood_group_cleansed', 'latitude', 'longitude', 
                   'property_type', 'room_type', 'accommodates', 'bathrooms',  
                   'bedrooms', 'bed_type', 'amenities', 'square_feet', 'price', 
                   'cleaning_fee', 'security_deposit', 'extra_people', 
                   'guests_included', 'minimum_nights', 'maximum_nights', 'instant_bookable', 
                   'is_business_travel_ready', 'cancellation_policy']

berlin_preprocess = berlin[columns_to_keep].set_index('id')
berlin_preprocess.head()
print(f'There is is {berlin_preprocess.shape[0]} lines and {berlin_preprocess.shape[1]} columns,'
      f' in this set after dropping irrelevant columns.')<jupyter_output>There is is 22552 lines and 26 columns, in this set after dropping irrelevant columns.<jupyter_text>Cleaning the price columns<jupyter_code># Reviewing all the monetary columns from this dataframe.

berlin_preprocess[['price', 'cleaning_fee', 'extra_people', 'security_deposit']].head()
# Checking how many missing values we have in each of these features.

print(berlin_preprocess.price.isna().sum())
print(berlin_preprocess.cleaning_fee.isna().sum())
print(berlin_preprocess.extra_people.isna().sum())
print(berlin_preprocess.security_deposit.isna().sum())
# We are making an assumption that missing values mean that landlords don't charge
# for those particular services. Exchaning missing values with $0.

berlin_preprocess['cleaning_fee'] = berlin_preprocess['cleaning_fee'].fillna('$0.00')
berlin_preprocess['security_deposit'] = berlin_preprocess['security_deposit'].fillna('$0.00')
# Wrangling strings in these features. Dropping $ signs and commas.

berlin_preprocess['price'] = berlin_preprocess['price'].str.replace('$', '').str.replace(',', '').astype(float)
berlin_preprocess['cleaning_fee'] = berlin_preprocess['cleaning_fee'].str.replace('$', '').str.replace(',', '').astype(float)
berlin_preprocess['extra_people'] = berlin_preprocess['extra_people'].str.replace('$', '').str.replace(',', '').astype(float)
berlin_preprocess['security_deposit'] = berlin_preprocess['security_deposit'].str.replace('$', '').str.replace(',', '').astype(float)
# Checking the layout of the price distribution.

berlin_preprocess['price'].describe()
# Graphical presentation of the above listed distribution.

square = dict(markerfacecolor='r', markeredgecolor='r', marker='.')
berlin_preprocess['price'].plot(kind='box', xlim=(0, 1000), vert=False, flierprops=square, figsize=(16, 2));
# Finishing initial preprocessing with dropping the values above $400 and equal $0.

berlin = berlin_preprocess.drop(berlin_preprocess[ (berlin_preprocess.price > 400) | (berlin_preprocess.price == 0) ].index, axis=0)
# Checking the shape after initial preprocessing.

berlin.shape<jupyter_output><empty_output><jupyter_text>Dealing with missing values<jupyter_code># Listing all the features with missing values.

berlin.isna().sum()
# Dropping the columns with large number of missing values

berlin = berlin.drop(columns=['space', 'host_response_time', 'host_response_rate', 'square_feet'])
# Checking the instances with missing bathroom values.

berlin.loc[berlin['bathrooms'].isna()]
# And with missing bedroom values.

berlin.loc[berlin['bedrooms'].isna()]
# It seems that most common value for bathroom is 1 so we are filling that value with nans.
# Using 0 bedrooms for missing bedroom values - most likely studio apartments.

berlin['bathrooms'] = berlin['bathrooms'].fillna(1.0)
berlin['bedrooms'] = berlin['bedrooms'].fillna(0.0)
# Finally filling out NaN on profile pic with f values.

berlin['host_has_profile_pic'] = berlin['host_has_profile_pic'].fillna(value='f')
# Checking our data after this second step of cleaning.

berlin.isna().sum()<jupyter_output><empty_output><jupyter_text>Baseline and initial model<jupyter_code># Actual baseline mean of the price.

berlin['price'].mean()<jupyter_output><empty_output><jupyter_text>Quick and dirty model using only numerical and categorical data<jupyter_code># Splitting our original dataframe into numerical and non numerical components.

berlin_numerical = berlin.select_dtypes(include='number')
berlin_non_numerical = berlin.select_dtypes(exclude='number')
berlin_numerical.head()<jupyter_output><empty_output><jupyter_text>Engineering couple of features<jupyter_code># Adding two features using numerical data only.
# One will show what would be an average cost per day at full occupancy.
# Second feature shows what is the minimum expense out of pocket if you decide
# to rent that specific space. 

#berlin_numerical['price_per_person'] = berlin_numerical['price'] / berlin_numerical['accommodates']
#berlin_numerical['minimum_expenditure'] = berlin_numerical['price'] * berlin_numerical['minimum_nights']

### Using these features created big leakeage into the data and produced r^2 scores of 0.98 - 1.
# Making sure that these engineered features are added properly.

berlin_numerical.head()<jupyter_output><empty_output><jupyter_text>Running the model on numerical data only<jupyter_code># Splitting the data into train, test groups.

X_train_num, X_test_num = train_test_split(berlin_numerical, train_size=0.8, test_size=0.2, random_state=42)

target = 'price'

y_train_num = X_train_num[target]
X_train_num = X_train_num.drop(columns=target)

y_test_num = X_test_num[target]
X_test_num = X_test_num.drop(columns=target)
# Training the model and getting the initial score.

berlin_num_pipeline = make_pipeline(
    SimpleImputer(),
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
)

berlin_num_pipeline.fit(X_train_num, y_train_num)

y_pred_num = berlin_num_pipeline.predict(X_test_num)
score_num = r2_score(y_test_num, y_pred_num)
print(f'Model based on numerical data only has a r^2 score of: {round(score_num, 2)}')
# Get feature importances from the initial numerical model.

rf = berlin_num_pipeline.named_steps['randomforestregressor']
importances = pd.Series(rf.feature_importances_, X_train_num.columns)

# Plot feature importances.

n = 20
plt.figure(figsize=(10, 10))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh();

# Based on this data I plan to drop all the data below 0.05 feature importance.
# In this case that would be bathrooms, maxium_nights and security_deposit.<jupyter_output><empty_output><jupyter_text>Running the model on categorical data only<jupyter_code>berlin_non_numerical.head()
# Dropping these text features as I'm unable to process them under the current deadline.

berlin_non_numerical = berlin_non_numerical.drop(columns=['description', 'amenities'])
# Clean up property_type column

berlin_non_numerical = berlin_non_numerical.replace(['Boutique hotel', 'Hotel',
                                                     'Bungalow', 'Boat', 'Tiny house',
                                                     'Houseboat', 'Camper/RV', 'Villa',
                                                     'Aparthotel', 'Pension (South Korea)',
                                                     'Cabin', 'Cottage', 'Resort',
                                                     'Train', 'Casa particular (Cuba)',
                                                     'Castle', 'Island', 'In-law', 'Chalet',
                                                     'Tipi', 'Cave', 'Barn'], 'Other')
# Clean up bed_type column

berlin_non_numerical = berlin_non_numerical.replace(['Pull-out Sofa', 'Futon', 'Couch', 'Airbed'], 'Sofa\Other')
# Clean up cancellation_policy column

berlin_non_numerical = berlin_non_numerical.replace(['strict_14_with_grace_period', 'super_strict_30', 'super_strict_60'], 'strict')

# Checking the cleaned non numerical dataframe.

berlin_non_numerical.head()
# Encoding the categorical values. 

encoder = ce.OneHotEncoder(use_cat_names=True)

encoded = encoder.fit_transform(berlin_non_numerical)
print(f'{len(encoded.columns)} columns')
encoded.head()
# Splitting the data into train, test groups.

X_train_encoded, X_test_encoded = train_test_split(encoded, train_size=0.8, test_size=0.2, random_state=42)

y_train_encoded = y_train_num
y_test_encoded = y_test_num
# Training the model and getting the initial score.

encoded_pipeline = make_pipeline(
    SimpleImputer(),
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
)

encoded_pipeline.fit(X_train_encoded, y_train_encoded)

y_pred_encoded = encoded_pipeline.predict(X_test_encoded)
score_encoded = r2_score(y_test_encoded, y_pred_encoded)
print(f'Encoded model based on categorical data has r^2 score: {round(score_encoded, 2)}')
# Get feature importances for categorical data.

rf = encoded_pipeline.named_steps['randomforestregressor']
importances = pd.Series(rf.feature_importances_, X_train_encoded.columns)

# Plot feature importances.

n = 20
plt.figure(figsize=(10, 10))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh();

# It seems that these are very fragmented. From the top 20 we will keep
# property_type, cancellation_policy and instant_bookable. 
# Neighbourhoud will be dropped since we are keeping the coordinates from the numerical set.<jupyter_output><empty_output><jupyter_text>Dropping the less important features and merging the final dataframe<jupyter_code># Dropping the unimoportant columns.

berlin_numerical = berlin_numerical.drop(columns=['bathrooms', 'maximum_nights', 'security_deposit'])
berlin_non_numerical = berlin_non_numerical.drop(columns=['neighbourhood_group_cleansed'])
# Encoding the categorical values. 

encoder = ce.OneHotEncoder(use_cat_names=True)

encoded = encoder.fit_transform(berlin_non_numerical)
print(f'{len(encoded.columns)} columns')
encoded.head()
# Creating the final dataframe.

berlin_final = pd.merge(berlin_numerical, encoded, left_index=True, right_index=True)<jupyter_output><empty_output><jupyter_text>Final model<jupyter_code>berlin_final.head()
X_train_final, X_test_final = train_test_split(berlin_final, train_size=0.8, test_size=0.2, random_state=42)
target_final = 'price'

y_train_final = X_train_final[target_final]
X_train_final = X_train_final.drop(columns=target_final)

y_test_final = X_test_final[target_final]
X_test_final = X_test_final.drop(columns=target_final)
berlin_pipeline_final = make_pipeline(
    SimpleImputer(),
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
)

berlin_pipeline_final.fit(X_train_final, y_train_final)

y_pred_final = berlin_pipeline_final.predict(X_test_final)
score_final = r2_score(y_test_final, y_pred_final)
print(f'Final model r^2 score: {round(score_final, 2)}')
# Get feature importances

rf = berlin_pipeline_final.named_steps['randomforestregressor']
importances = pd.Series(rf.feature_importances_, X_train_final.columns)

# Plot feature importances

n = 20
plt.figure(figsize=(10, 10))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10478428993858518, 'fraction_numerical': 0.010994010159981802, 'mean_word_length': 5.53941497273178, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 48, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '343593', 'license': 'MIT', 'n_tokens_mistral': 4280, 'n_tokens_neox': 4120, 'n_words': 1013, 'path': 'Airbnb.ipynb', 'repo_name': 'BuildWeekAirbnbOptimal2/DataScience', 'size': '13189'}