<jupyter_start><jupyter_text>Testing different conditions for regression<jupyter_code>import numpy as np
import pandas as pd
import seaborn as sns
from re import sub
from decimal import Decimal
from matplotlib import pyplot as plt
import datetime
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from nltk import word_tokenize, WordNetLemmatizer
from sklearn.model_selection import train_test_split, cross_validate, GridSearchCV
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.preprocessing import Imputer, StandardScaler
from sklearn.decomposition import PCA
from math import sqrt
import tests as t
import re
import nltk
from sklearn.preprocessing import OneHotEncoder
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
from sklearn.linear_model import SGDRegressor
import folium
from sklearn.pipeline import Pipeline
from sklearn import linear_model

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
import xgboost as xgb
pd.set_option('display.max_columns', 106)
from sklearn.metrics import mean_squared_error, mean_absolute_error

import collections
from sklearn import metrics
from sklearn import datasets
from sklearn import linear_model
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
from collections import Counter
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
from collections import Counter

import matplotlib.pyplot as plt
from pylab import *
porto = pd.read_csv(r"C:\Users\sousa\Desktop\github\Airbnb Tale of 2 cities\Data Portugal\porto_listings.csv")
lisbon = pd.read_csv(r"C:\Users\sousa\Desktop\github\Airbnb Tale of 2 cities\Data Portugal\lisbon_listings.csv")

porto_min = pd.read_csv(r"C:\Users\sousa\Desktop\github\Airbnb Tale of 2 cities\Data Portugal\porto_listings_min.csv")
lisbon_min = pd.read_csv(r"C:\Users\sousa\Desktop\github\Airbnb Tale of 2 cities\Data Portugal\lisbon_listings_min.csv")
porto.describe()
lisbon.describe()<jupyter_output><empty_output><jupyter_text>Dimensionality reduction, determining min components<jupyter_code>#function to scale and normalize the dataframes features
def feature_scaling(df, type_scale):
    '''
    This function takes in either the azdias or the customers dataframe and applyes the selected feature scaler
    Args: customer or azdias dataframe and a string representing the type of scaling intended
    returns: scaled dataframe
    '''
    
    features_list = df.columns
    
    if type_scale == 'StandardScaler':
        df_scaled = StandardScaler().fit_transform(df)
        
    if type_scale == 'MinMaxScaler':
        df_scaled = MinMaxScaler().fit_transform(df)
    
    df_scaled = pd.DataFrame(df_scaled)
    df_scaled.columns = features_list
    
    return df_scaled
#pca model
def pca_model(df, n_components):
    '''
    This function defines a model that takes in a previously scaled dataframe and returns the result of 
    the transformation. The output is an onject created post data fitting
    '''
    pca = PCA(n_components)
    pca_df = pca.fit(df)
    
    return pca_df
#scree plots for PCA
def scree_plots(SS1,SS2, dataname1, dataname2):
    '''
    This function takes in the transformed data using PCA and plots it in scree plots
    '''
    subplot(2,1,1)

    plt.plot(np.cumsum(SS1.explained_variance_ratio_))
    plt.xlabel('Number of Components')
    plt.ylabel('Explained Variance Ratio')
    plt.title('Explained Variance Ratio vs Number of Components SS' + dataname1)
    plt.grid(b=True)

    subplot(2,1,2)
    plt.plot(np.cumsum(SS2.explained_variance_ratio_))
    plt.xlabel('Number of Components')
    plt.ylabel('Explained Variance Ratio')
    plt.title('Explained Variance Ratio vs Number of Components SS' + dataname2)
    plt.grid(b=True)

    plot = tight_layout()
    plot = plt.show()
porto_SS = feature_scaling(porto, 'StandardScaler')
lisbon_SS = feature_scaling(lisbon, 'StandardScaler')

n_components_porto = len(porto_SS.columns.values)
n_components_lisbon = len(lisbon_SS.columns.values)

porto_SS_pca = pca_model(porto_SS, n_components_porto)
lisbon_SS_pca = pca_model(lisbon_SS, n_components_lisbon)

scree_plots(porto_SS_pca, lisbon_SS_pca, ' Porto', ' Lisbon')<jupyter_output><empty_output><jupyter_text>Using standard scaler with 150 principal components 90% of the original variance can be represented  Linear Regression<jupyter_code>def LR_comparisons(df, city):
    
    #getting target
    X = df.drop('price', 1)
    y = df['price']
    
    #split data
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    
    #no scaling and no dimensionality reduction
    no_scale_no_pca = Pipeline([('linear', linear_model.LinearRegression())])
    no_scale_no_pca.fit(X_train, y_train)
    y_pred1 = no_scale_no_pca.predict(X_test)
    linear_reg_err1 = metrics.median_absolute_error(y_test, y_pred1)
    linear_reg_r21 = metrics.r2_score(y_test, y_pred1)


    #with scaling, no dimensionality reduction
    scale_no_pca = Pipeline([('standardize', StandardScaler()), ('linear', linear_model.LinearRegression())])
    scale_no_pca.fit(X_train, y_train)
    y_pred2 = scale_no_pca.predict(X_test)
    linear_reg_err2 = metrics.median_absolute_error(y_test, y_pred2)
    linear_reg_r22 = metrics.r2_score(y_test, y_pred2)


    #scaling and dimensionality reduction
    scale_pca = Pipeline([('standardize', StandardScaler()),
        ('pca', PCA(n_components = 125)), ('linear', linear_model.LinearRegression())])
    scale_pca.fit(X_train, y_train)
    y_pred3 = scale_pca.predict(X_test)
    linear_reg_err3 = metrics.median_absolute_error(y_test, y_pred3)
    linear_reg_r23 = metrics.r2_score(y_test, y_pred3)


    print ("Linear Regression's MAE without PCA or scaling in " + city + " : " + str(linear_reg_err1))
    print ("Linear Regression's MAE deviation with scaling in " + city + " : " + str(linear_reg_err2))
    print ("Linear Regression's MAE deviation with PCA and scaling in " + city + " : " + str(linear_reg_err3))

    print ("Linear Regression's r2 without PCA or scaling in " + city + " : "+ str(linear_reg_r21))
    print ("Linear Regression's r2 deviation with scaling in " + city + " : "+ str(linear_reg_r22))
    print ("Linear Regression's r2 deviation with PCA and scaling in " + city + " : " + str(linear_reg_r23))
    
LR_comparisons(porto, 'porto')
LR_comparisons(lisbon, 'lisbon')
LR_comparisons(porto_min, 'porto min')
LR_comparisons(lisbon_min, 'lisbon min')
X_porto = porto.drop('price', 1)
y_porto = porto['price']

X_lisbon = lisbon.drop('price', 1)
y_lisbon = lisbon['price']
# split our data
X_porto_train, X_porto_test, y_porto_train, y_porto_test = train_test_split(X_porto, y_porto, test_size=0.2)

X_lisbon_train, X_lisbon_test, y_lisbon_train, y_lisbon_test = train_test_split(X_lisbon, y_lisbon, test_size=0.2)<jupyter_output><empty_output><jupyter_text>Linear Regression with no scaling, scalling, with pca and without pca<jupyter_code>#no scaling and no dimensionality reduction
no_scale_no_pca = Pipeline([('linear', linear_model.LinearRegression())])
no_scale_no_pca.fit(X_porto_train, y_porto_train)
y_pred1_porto = no_scale_no_pca.predict(X_porto_test)
linear_reg_err1_porto = metrics.median_absolute_error(y_porto_test, y_pred1_porto)
linear_reg_r21_porto = metrics.r2_score(y_porto_test, y_pred1_porto)


#with scaling, no dimensionality reduction
scale_no_pca = Pipeline([('standardize', StandardScaler()), ('linear', linear_model.LinearRegression())])
scale_no_pca.fit(X_porto_train, y_porto_train)
y_pred2_porto = scale_no_pca.predict(X_porto_test)
linear_reg_err2_porto = metrics.median_absolute_error(y_porto_test, y_pred2_porto)
linear_reg_r22_porto = metrics.r2_score(y_porto_test, y_pred2_porto)

    
#scaling and dimensionality reduction
scale_pca = Pipeline([('standardize', StandardScaler()),
    ('pca', PCA(n_components = 125)), ('linear', linear_model.LinearRegression())])
scale_pca.fit(X_porto_train, y_porto_train)
y_pred3_porto = scale_pca.predict(X_porto_test)
linear_reg_err3_porto = metrics.median_absolute_error(y_porto_test, y_pred3_porto)
linear_reg_r23_porto = metrics.r2_score(y_porto_test, y_pred3_porto)


print ("Linear Regression's MAE without PCA or scaling in Porto: " + str(linear_reg_err1_porto))
print ("Linear Regression's MAE deviation with scaling in Porto: " + str(linear_reg_err2_porto))
print ("Linear Regression's MAE deviation with PCA and scaling in Porto: " + str(linear_reg_err3_porto))

print ("Linear Regression's r2 without PCA or scaling in Porto: " + str(linear_reg_r21_porto))
print ("Linear Regression's r2 deviation with scaling in Porto: " + str(linear_reg_r22_porto))
print ("Linear Regression's r2 deviation with PCA and scaling in Porto: " + str(linear_reg_r23_porto))

plt.scatter(y_porto_test, y_pred1_porto, c = 'orange')
#no scaling and no dimensionality regression
no_scale_no_pca = Pipeline([('linear', linear_model.LinearRegression())])
no_scale_no_pca.fit(X_lisbon_train, y_lisbon_train)
y_pred1_lisbon = no_scale_no_pca.predict(X_lisbon_test)
linear_reg_err1_lisbon = metrics.median_absolute_error(y_lisbon_test, y_pred1_lisbon)
linear_reg_r21_lisbon = metrics.r2_score(y_lisbon_test, y_pred1_lisbon)


#with scaling, no dimensionality regression
scale_no_pca = Pipeline([('standardize', StandardScaler()), ('linear', linear_model.LinearRegression())])
scale_no_pca.fit(X_lisbon_train, y_lisbon_train)
y_pred2_lisbon = scale_no_pca.predict(X_lisbon_test)
linear_reg_err2_lisbon = metrics.median_absolute_error(y_lisbon_test, y_pred2_lisbon)
linear_reg_r22_lisbon = metrics.r2_score(y_lisbon_test, y_pred2_lisbon)

    
#scaling and dimensionality reduction
scale_pca = Pipeline([('standardize', StandardScaler()),
    ('pca', PCA(n_components = 125)), ('linear', linear_model.LinearRegression())])
scale_pca.fit(X_lisbon_train, y_lisbon_train)
y_pred3_lisbon = scale_pca.predict(X_lisbon_test)
linear_reg_err3_lisbon = metrics.median_absolute_error(y_lisbon_test, y_pred3_lisbon)
linear_reg_r23_lisbon = metrics.r2_score(y_lisbon_test, y_pred3_lisbon)


print ("Linear Regression's MAE without PCA or scaling in Lisbon: " + str(linear_reg_err1_lisbon))
print ("Linear Regression's MAE deviation with scaling in Lisbon: " + str(linear_reg_err2_lisbon))
print ("Linear Regression's MAE deviation with PCA and scaling in Lisbon: " + str(linear_reg_err3_lisbon))
print ("Linear Regression's r2 without PCA or scaling in Lisbon: " + str(linear_reg_r21_lisbon))
print ("Linear Regression's r2 deviation with scaling in Lisbon: " + str(linear_reg_r22_lisbon))
print ("Linear Regression's r2 deviation with PCA and scaling in Lisbon: " + str(linear_reg_r23_lisbon))<jupyter_output><empty_output><jupyter_text>HPM model<jupyter_code>hpm_reg_start = time.time()

# Create instance of the model, `LinearRegression` function from 
# Scikit-Learn and fit the model on the training data:

hpm_reg = LinearRegression()  
hpm_reg.fit(X_porto_train, y_porto_train) #training the algorithm

# Now that the model has been fit we can make predictions by calling 
# the predict command. We are making predictions on the testing set:
training_preds_hpm_reg = hpm_reg.predict(X_porto_train)
val_preds_hpm_reg = hpm_reg.predict(X_porto_test)

hpm_reg_end = time.time()

print(f"Time taken to run: {round((hpm_reg_end - hpm_reg_start)/60,1)} minutes")

# Check the predictions against the actual values by using the MSE and R-2 metrics:
print("\nTraining RMSE:", round(mean_squared_error(y_porto_train, training_preds_hpm_reg),4))
print("Validation RMSE:", round(mean_squared_error(y_porto_test, val_preds_hpm_reg),4))
print("\nTraining r2:", round(r2_score(y_porto_train, training_preds_hpm_reg),4))
print("Validation r2:", round(r2_score(y_porto_test, val_preds_hpm_reg),4))
y_test_array = np.array(list(y_porto_test))
val_preds_hpm_reg_array = np.array(val_preds_hpm_reg)
hpm_df = pd.DataFrame({'Actual': y_test_array.flatten(), 'Predicted': val_preds_hpm_reg_array.flatten()})
hpm_df<jupyter_output><empty_output><jupyter_text>Ridge Regularization<jupyter_code>lr = linear_model.LinearRegression()

for i in range (-2, 3):
    alpha = 10**i
    rm = linear_model.Ridge(alpha=alpha)
    ridge_model = rm.fit(X_porto_train, y_porto_train)
    preds_ridge = ridge_model.predict(X_porto_test)

    plt.scatter(preds_ridge, y_porto_test, alpha=.75, color='r')
    plt.xlabel('Predicted Price')
    plt.ylabel('Actual Price')
    plt.title('Ridge Regularization with alpha = {}'.format(alpha))
    overlay = 'R^2 is: {}\nRMSE is: {}'.format(
                   round(ridge_model.score(X_porto_test, y_porto_test), 4),
                    round(mean_squared_error(y_porto_train, training_preds_hpm_reg),4))
    plt.annotate( s=overlay,xy=(5.5,2.5),size='x-large')
    plt.show()<jupyter_output><empty_output><jupyter_text>Xgboost Regressor no scaling<jupyter_code>booster = xgb.XGBRegressor()

param_grid = {'n_estimators': [100, 200, 300],
              'learning_rate': [0.01, 0.05, 0.1], 
              'max_depth': [3, 4, 5, 6, 7],
              'colsample_bytree': [0.6, 0.7, 1],
              'gamma': [0.0, 0.1, 0.2]}

booster_grid_search = GridSearchCV(booster, param_grid, cv=3, n_jobs=-1)
# train the tuned random forest porto
booster_grid_search.fit(X_porto_train, y_porto_train)

print(booster_grid_search.best_params_)
# train the tuned random forest lisbon
booster_grid_search.fit(X_lisbon_train, y_lisbon_train)

print(booster_grid_search.best_params_)
#instantiating the models based on the hyperparameters found by grid search
booster_porto = xgb.XGBRegressor(colsample_bytree=0.7, gamma=0.1, learning_rate=0.1, 
                           max_depth=3, n_estimators=300, random_state=4)

booster_lisbon = xgb.XGBRegressor(colsample_bytree=0.7, gamma=0.2, learning_rate=0.1, 
                           max_depth=6, n_estimators=300, random_state=4)
#training
booster_porto.fit(X_porto_train, y_porto_train)

booster_lisbon.fit(X_lisbon_train, y_lisbon_train)
#prediction
y_pred_train_porto = booster_porto.predict(X_porto_train)
y_pred_test_porto = booster_porto.predict(X_porto_test)

y_pred_train_lisbon = booster_lisbon.predict(X_lisbon_train)
y_pred_test_lisbon = booster_lisbon.predict(X_lisbon_test)
#metrics to determine quality of model
RMSE_porto = np.sqrt(mean_squared_error(y_porto_test, y_pred_test_porto))
RMSE_lisbon = np.sqrt(mean_squared_error(y_lisbon_test, y_pred_test_lisbon))

MSE_porto = mean_squared_error(y_porto_test, y_pred_test_porto)
MSE_lisbon = mean_squared_error(y_lisbon_test, y_pred_test_lisbon)

r2_porto = r2_score(y_porto_test, y_pred_test_porto)
r2_lisbon = r2_score(y_lisbon_test, y_pred_test_lisbon)



print(f"RMSE_porto: {round(RMSE_porto, 4)}")
print(f"RMSE_lisbon: {round(RMSE_lisbon, 4)}")

print(f"MSE_porto: {round(MSE_porto, 4)}")
print(f"MSE_lisbon: {round(MSE_lisbon, 4)}")

print(f"r2_porto: {round(r2_porto, 4)}")
print(f"r2_lisbon: {round(r2_lisbon, 4)}")<jupyter_output><empty_output><jupyter_text>XGboost with scaling<jupyter_code>X_porto_SS = porto_SS.drop('price', 1)
y_porto_SS = porto_SS['price']

X_lisbon_SS = lisbon_SS.drop('price', 1)
y_lisbon_SS = lisbon_SS['price']
# split our data
X_porto_train, X_porto_test, y_porto_train, y_porto_test = train_test_split(X_porto_SS, y_porto_SS, test_size=0.2)

X_lisbon_train, X_lisbon_test, y_lisbon_train, y_lisbon_test = train_test_split(X_lisbon_SS, y_lisbon_SS, test_size=0.2)
booster = xgb.XGBRegressor()

param_grid = {'n_estimators': [100, 200, 300],
              'learning_rate': [0.01, 0.05, 0.1], 
              'max_depth': [3, 4, 5, 6, 7],
              'colsample_bytree': [0.6, 0.7, 1],
              'gamma': [0.0, 0.1, 0.2]}

booster_grid_search = GridSearchCV(booster, param_grid, cv=3, n_jobs=-1)
# train the tuned random forest porto
booster_grid_search.fit(X_porto_train, y_porto_train)

print(booster_grid_search.best_params_)
# train the tuned random forest lisbon
booster_grid_search.fit(X_lisbon_train, y_lisbon_train)

print(booster_grid_search.best_params_)
#instantiating the models based on the hyperparameters found by grid search
booster_porto = xgb.XGBRegressor(colsample_bytree = 1, gamma = 0.0, learning_rate = 0.1, 
                           max_depth= 3, n_estimators = 300, random_state = 4)

booster_lisbon = xgb.XGBRegressor(colsample_bytree = 1, gamma = 0.0, learning_rate = 0.05, 
                           max_depth = 4, n_estimators = 300, random_state = 4)
#training
booster_porto.fit(X_porto_train, y_porto_train)

booster_lisbon.fit(X_lisbon_train, y_lisbon_train)
#prediction
y_pred_train_porto = booster_porto.predict(X_porto_train)
y_pred_test_porto = booster_porto.predict(X_porto_test)

y_pred_train_lisbon = booster_lisbon.predict(X_lisbon_train)
y_pred_test_lisbon = booster_lisbon.predict(X_lisbon_test)
#metrics to determine quality of model
RMSE_porto = np.sqrt(mean_squared_error(y_porto_test, y_pred_test_porto))
RMSE_lisbon = np.sqrt(mean_squared_error(y_lisbon_test, y_pred_test_lisbon))

MSE_porto = mean_squared_error(y_porto_test, y_pred_test_porto)
MSE_lisbon = mean_squared_error(y_lisbon_test, y_pred_test_lisbon)

r2_porto = r2_score(y_porto_test, y_pred_test_porto)
r2_lisbon = r2_score(y_lisbon_test, y_pred_test_lisbon)



print(f"RMSE_porto: {round(RMSE_porto, 4)}")
print(f"RMSE_lisbon: {round(RMSE_lisbon, 4)}")

print(f"MSE_porto: {round(MSE_porto, 4)}")
print(f"MSE_lisbon: {round(MSE_lisbon, 4)}")

print(f"r2_porto: {round(r2_porto, 4)}")
print(f"r2_lisbon: {round(r2_lisbon, 4)}")<jupyter_output><empty_output><jupyter_text>In the Feature selection notebook I came up with a list of features that could be good candidates for reduced data approach to modeling  Modeling with Feature selection<jupyter_code>lisbon_list = ['last_review_present',
 'neighbourhood_cleansed_Aldeia Galega da Merceana e Aldeia Gavinha',
 'accommodates',
 'host_name_present',
 'price_per_guest',
 'neighbourhood_cleansed_Arruda dos Vinhos',
 'neighbourhood_cleansed_Cardosas',
 'neighbourhood_cleansed_Arranh',
 'neighbourhood_group_cleansed_Arruda Dos Vinhos',
 'neighbourhood_cleansed_Mina de gua',
 'neighbourhood_cleansed_Falagueira-Venda Nova',
 'neighbourhood_cleansed_Alfragide',
 'neighbourhood_cleansed_Venteira',
 'neighbourhood_group_cleansed_Amadora',
 'neighbourhood_cleansed_guas Livres',
 'neighbourhood_cleansed_Sapataria',
 'neighbourhood_cleansed_Santo Quintino',
 'is_business_travel_ready',
 'neighbourhood_cleansed_Sobral de Monte Agrao',
 'neighbourhood_group_cleansed_Sobral De Monte Agrao',
 'neighbourhood_cleansed_Encosta do Sol',
 'reviews_p_month_present',
 'neighbourhood_cleansed_Alcoentre',
 'neighbourhood_cleansed_Pvoa de Santa Iria e Forte da Casa',
 'neighbourhood_cleansed_Alhandra, So Joo dos Montes e Calhandriz',
 'neighbourhood_group_cleansed_Azambuja',
 'neighbourhood_group_cleansed_Vila Franca De Xira',
 'neighbourhood_cleansed_Castanheira do Ribatejo e Cachoeiras',
 'neighbourhood_cleansed_Manique do Intendente, V.N.de S.Pedro e Maussa',
 'neighbourhood_cleansed_Vialonga',
 'neighbourhood_cleansed_Vila Franca de Xira',
 'neighbourhood_cleansed_Azambuja',
 'neighbourhood_cleansed_Vale do Paraso',
 'neighbourhood_cleansed_Alverca do Ribatejo e Sobralinho']

porto_list = ['reviews_p_month_present',
 'host_since_present',
 'property_type_Windmill',
 'last_review_present',
 'neighbourhood_cleansed_São João de Ver',
 'neighbourhood_cleansed_Campo e Sobrado',
 'price_per_guest',
 'accommodates',
 'first_review_present',
 'has_availability',
 'neighbourhood_cleansed_Roge',
 'neighbourhood_cleansed_São Pedro de Castelões',
 'neighbourhood_cleansed_Cepelos',
 'neighbourhood_cleansed_Caldas de São Jorge e de Pigeiros',
 'neighbourhood_cleansed_Fiães',
 'neighbourhood_cleansed_Romariz',
 'neighbourhood_group_cleansed_SANTA MARIA DA FEIRA',
 'neighbourhood_cleansed_Arões',
 'neighbourhood_cleansed_Macieira de Cambra',
 'neighbourhood_group_cleansed_VALE DE CAMBRA',
 'neighbourhood_group_cleansed_SÃO JOÃO DA MADEIRA',
 'neighbourhood_cleansed_Sanguedo',
 'neighbourhood_cleansed_Santa Maria da Feira, Travanca, Sanfins e Espargo',
 'neighbourhood_cleansed_Lourosa',
 'neighbourhood_cleansed_Lobão, Gião, Louredo e Guisande',
 'neighbourhood_cleansed_São João da Madeira',
 'neighbourhood_cleansed_Canedo, Vale e Vila Maior',
 'neighbourhood_cleansed_Argoncilhe',
 'neighbourhood_cleansed_Bougado (São Martinho e Santiago)',
 'neighbourhood_cleansed_Coronado (São Romão e São Mamede)',
 'neighbourhood_cleansed_Mozelos',
 'neighbourhood_cleansed_Muro',
 'is_business_travel_ready',
 'neighbourhood_cleansed_Alvarelhos e Guidões',
 'neighbourhood_cleansed_Covelas',
 'neighbourhood_group_cleansed_TROFA',
 'neighbourhood_cleansed_Ermesinde',
 'neighbourhood_cleansed_Paredes',
 'neighbourhood_cleansed_Gandra',
 'neighbourhood_cleansed_Alfena',
 'neighbourhood_cleansed_Rebordosa',
 'neighbourhood_cleansed_Cete',
 'neighbourhood_cleansed_Aguiar de Sousa',
 'neighbourhood_group_cleansed_VALONGO',
 'neighbourhood_group_cleansed_PAREDES',
 'neighbourhood_cleansed_Baltar',
 'host_name_present',
 'neighbourhood_cleansed_Valongo',
 'neighbourhood_cleansed_Sobreira',
 'neighbourhood_cleansed_Parada de Todeia']
X_porto_f = porto[porto_list]
y_porto_f = porto.price.values

X_lisbon_f = lisbon[lisbon_list]
y_lisbon_f = lisbon.price.values<jupyter_output><empty_output><jupyter_text>Lets run the models with the selected features<jupyter_code># split our data
X_porto_train, X_porto_test, y_porto_train, y_porto_test = train_test_split(X_porto_f, y_porto_f, test_size=0.33)

X_lisbon_train, X_lisbon_test, y_lisbon_train, y_lisbon_test = train_test_split(X_lisbon_f, y_lisbon_f, test_size=0.33)
#no scaling and no dimensionality reduction
no_scale_no_pca = Pipeline([('linear', linear_model.LinearRegression())])
no_scale_no_pca.fit(X_porto_train, y_porto_train)
y_pred1_porto = no_scale_no_pca.predict(X_porto_test)
linear_reg_err1_porto = metrics.median_absolute_error(y_porto_test, y_pred1_porto)
linear_reg_r21_porto = metrics.r2_score(y_porto_test, y_pred1_porto)


#with scaling, no dimensionality reduction
scale_no_pca = Pipeline([('standardize', StandardScaler()), ('linear', linear_model.LinearRegression())])
scale_no_pca.fit(X_porto_train, y_porto_train)
y_pred2_porto = scale_no_pca.predict(X_porto_test)
linear_reg_err2_porto = metrics.median_absolute_error(y_porto_test, y_pred2_porto)
linear_reg_r22_porto = metrics.r2_score(y_porto_test, y_pred2_porto)

    
#scaling and dimensionality reduction
scale_pca = Pipeline([('standardize', StandardScaler()),
    ('pca', PCA(n_components = 20)), ('linear', linear_model.LinearRegression())])
scale_pca.fit(X_porto_train, y_porto_train)
y_pred3_porto = scale_pca.predict(X_porto_test)
linear_reg_err3_porto = metrics.median_absolute_error(y_porto_test, y_pred3_porto)
linear_reg_r23_porto = metrics.r2_score(y_porto_test, y_pred3_porto)


print ("Linear Regression's MAE without PCA or scaling in Porto: " + str(linear_reg_err1_porto))
print ("Linear Regression's MAE deviation with scaling in Porto: " + str(linear_reg_err2_porto))
print ("Linear Regression's MAE deviation with PCA and scaling in Porto: " + str(linear_reg_err3_porto))

print ("Linear Regression's r2 without PCA or scaling in Porto: " + str(linear_reg_r21_porto))
print ("Linear Regression's r2 deviation with scaling in Porto: " + str(linear_reg_r22_porto))
print ("Linear Regression's r2 deviation with PCA and scaling in Porto: " + str(linear_reg_r23_porto))
#no scaling and no dimensionality regression
no_scale_no_pca = Pipeline([('linear', linear_model.LinearRegression())])
no_scale_no_pca.fit(X_lisbon_train, y_lisbon_train)
y_pred1_lisbon = no_scale_no_pca.predict(X_lisbon_test)
linear_reg_err1_lisbon = metrics.median_absolute_error(y_lisbon_test, y_pred1_lisbon)
linear_reg_r21_lisbon = metrics.r2_score(y_lisbon_test, y_pred1_lisbon)


#with scaling, no dimensionality regression
scale_no_pca = Pipeline([('standardize', StandardScaler()), ('linear', linear_model.LinearRegression())])
scale_no_pca.fit(X_lisbon_train, y_lisbon_train)
y_pred2_lisbon = scale_no_pca.predict(X_lisbon_test)
linear_reg_err2_lisbon = metrics.median_absolute_error(y_lisbon_test, y_pred2_lisbon)
linear_reg_r22_lisbon = metrics.r2_score(y_lisbon_test, y_pred2_lisbon)

    
#scaling and dimensionality reduction
scale_pca = Pipeline([('standardize', StandardScaler()),
    ('pca', PCA(n_components = 20)), ('linear', linear_model.LinearRegression())])
scale_pca.fit(X_lisbon_train, y_lisbon_train)
y_pred3_lisbon = scale_pca.predict(X_lisbon_test)
linear_reg_err3_lisbon = metrics.median_absolute_error(y_lisbon_test, y_pred3_lisbon)
linear_reg_r23_lisbon = metrics.r2_score(y_lisbon_test, y_pred3_lisbon)


print ("Linear Regression's MAE without PCA or scaling in Lisbon: " + str(linear_reg_err1_lisbon))
print ("Linear Regression's MAE deviation with scaling in Lisbon: " + str(linear_reg_err2_lisbon))
print ("Linear Regression's MAE deviation with PCA and scaling in Lisbon: " + str(linear_reg_err3_lisbon))
print ("Linear Regression's r2 without PCA or scaling in Lisbon: " + str(linear_reg_r21_lisbon))
print ("Linear Regression's r2 deviation with scaling in Lisbon: " + str(linear_reg_r22_lisbon))
print ("Linear Regression's r2 deviation with PCA and scaling in Lisbon: " + str(linear_reg_r23_lisbon))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09516622160300321, 'fraction_numerical': 0.015209566933704864, 'mean_word_length': 6.914241960183767, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '364388', 'license': 'CNRI-Python', 'n_tokens_mistral': 10191, 'n_tokens_neox': 9584, 'n_words': 1852, 'path': 'Data_Modeling.ipynb', 'repo_name': 'sousablde/Airbnb-A-tale-of-2-cities', 'size': '25839'}