<jupyter_start><jupyter_text>Key Takeaways from Initial Data Analysis- There don't seem to be any consistencies with the missing bathroom/bedroom data. I believe it was simply never input into the airbnb listing. Therefore, I chose to delete out those listings.- Almost all (15,827 out of 15,835) "last_review" and "first review" are because there are no reviews. These columns are dates. Cut these columns since "host_since" had less missing data and essentially fills the same aspect of understandinig how long the airbnb has been around for.- All null "review_scores_rating" are caused by no reviews. Therefore, I've inserted "no reviews" into column- Almost all of the reviews contain images. This could make a cool project to analyze how thumbnail images affect price. Most likely not useful for our project.- Created new columns based on extracted text from "ammenities" column- For 180 NaN's in "host has no profile picture." I decided to just assumed there was no image and set value to false<jupyter_code># Import Statements
from datetime import datetime
import pandas as pd
import numpy as np
from PIL import Image
import requests
import datetime
from datetime import datetime
# Import files
train = pd.read_csv("data/train.csv", index_col='id')
#test = pd.read_csv("archive-2/test.csv", index_col='id')
train_before = train.shape
def cleaned_dataframe(df):
    """
    1. Adds feature columns to df
    2. Deals with all null values
    3. Turns ratings into categorical column
    4. Converts "Host_since" into a measure of time
    """ 
    features = ['Wireless Internet','Air conditioning', 'Kitchen', 'Heating','Family/kid friendly', 'Essentials', 'Hair dryer', 'Iron', 
                'Smoke detector', 'Shampoo', 'Hangers', 'Hair dryer', 'Fire extinguisher', 'Laptop friendly workspace', 'First aid kit', 'Indoor fireplace',
                'TV','Cable TV', 'Elevator in building']
    
    # forloop to create all new columns
    for item in features:
        df[item]=np.where(df['amenities'].str.contains(item), 1, 0)
        
        
    # drop unnecessary column & columns with no host information
    # neighborhood will be dictated by zip and latitude/longitude
    df.drop(columns=['amenities', 'first_review', 'last_review', 'host_response_rate', 'neighbourhood'], axis=1, inplace=True)
    
    # drop rows with null values in certain columns
    df = df.dropna(axis=0, subset=['bathrooms', 'bedrooms', 'beds'])
    
    # dropped rows with no host information or now zip code
    df = df.dropna(axis=0, subset=['host_since', 'host_identity_verified', 'zipcode'])
    
    # Dealing with ratings column
    # Zero isn't a real rating in the columns
    # Temporarily assign rating_score with no previous reviews as 0 so it can later make it a category
    df['review_scores_rating']=np.where(df['number_of_reviews']==0, 0, df['review_scores_rating'])
    
    # drop remaining 800 rows with no values
    df = df.dropna(axis=0, subset=['review_scores_rating'])
    
    # change reviews into categories
    df['review_scores_rating'] = df['review_scores_rating'].round(-1).astype('int').astype('str')
    
    # reassign 0 ratings as "no past ratings" category
    df['review_scores_rating']=np.where(df['number_of_reviews']==0, 'no past ratings', df['review_scores_rating'])
    
    # Convert "host_since" into column that measures # of days an individual has been a host
    for i in range(len(df['host_since'])):
        today = datetime.today()
        date_time_obj = datetime.strptime(df['host_since'].iloc[i], '%Y-%m-%d')
        df['host_since'].iloc[i] = (today - date_time_obj).days
    
    # Convert "host_since" from object to int
    df['host_since'] = df['host_since'].astype('int')
    
    # True/false columns to 1/0
    df['cleaning_fee']=np.where(df['cleaning_fee']==True, 1, 0)
    df['host_has_profile_pic']=np.where(df['host_has_profile_pic']=='t', 1, 0)
    df['host_identity_verified']=np.where(df['host_identity_verified']=='t', 1, 0)
    df['instant_bookable']=np.where(df['instant_bookable']=='t', 1, 0)
    
    #drop columns with low correlation
    df.drop(columns=['latitude', 'longitude', 'Smoke detector', 'number_of_reviews', 'Hangers','First aid kit', 'Elevator in building', 'Essentials', 'zipcode', 'thumbnail_url', 'description', 'name'], axis=1, inplace=True)
    return df
train = cleaned_dataframe(train)
# Statement on cut data
train_after = train.shape
lost_rows = train_before[0]-train_after[0]
print("We cut", lost_rows, "data points when cleaning data leaving",train_after[0],"data points.")
# Each column contains image of airbnb

# def image_look(pick_a_row):
#     url = train['thumbnail_url'].iloc[pick_a_row]
#     im = Image.open(requests.get(url, stream=True).raw)
#     return im

# image_look(12304)
# Export CSV Files
train.to_csv('train_2.csv')
test.to_csv('test_2.csv')<jupyter_output><empty_output><jupyter_text>The Models<jupyter_code>train.head(1)
train.dtypes
train.corr()
train['review_scores_rating']
import seaborn as sns
#sns.pairplot(train)
test.columns
test.dtypes
# IMPORTS FOR MODEL
from sklearn.metrics import mean_absolute_error
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.compose import ColumnTransformer
from sklearn.linear_model import LinearRegression
from sklearn.feature_selection import SelectKBest, VarianceThreshold
from sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.model_selection import train_test_split
test.head(1)
# Split into X, y, train/test

target = 'log_price'
y = train[target]
X = train.drop(columns=target)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
# Linear Regression Baseline
y_mean = [y_train.mean()] * len(y_train)

baseline_mae = mean_absolute_error(y_train, y_mean)

print('Baseline MAE:', baseline_mae)
# Categorical columns to OneHotEncode
cat = X_train.select_dtypes(include=['object', 'bool']).columns

# Integer/Float Columns to Scale
num = X_train.select_dtypes(include=['int', 'float']).columns
# sanity check
assert len(cat) + len(num) == train.shape[1]-1, 'Column categories don\'t equal df columns'
model_lr = make_pipeline(
    OneHotEncoder(handle_unknown='ignore'),
    LinearRegression()
)

model_lr.fit(X_train, y_train);
train_mae = mean_absolute_error(y_train, model_lr.predict(X_train))
test_mae = mean_absolute_error(y_test, model_lr.predict(X_test))
score = model_lr.score(X_train, y_train)

print('Linear Regression Model')
print('Training MAE:', train_mae)
print('Validation MAE:', test_mae)
print('R^2 Score:', score)<jupyter_output>Linear Regression Model
Training MAE: 0.32850936553567783
Validation MAE: 0.3515381903885942
R^2 Score: 0.6297182783267588<jupyter_text>Linear Regression<jupyter_code># define model
model = LinearRegression()
# define transform
transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore'), cat),
                                             ('num', StandardScaler(), num)], remainder='passthrough')
# define pipeline
pipeline = Pipeline(steps=[('t', transformer), ('m',model)])
# fit the pipeline on the transformed data
pipeline.fit(X_train, y_train)
# make predictions
yhat = pipeline.predict(X_test)
#Gradient Boosting Model
train_mae = mean_absolute_error(y_train, pipeline.predict(X_train))
test_mae = mean_absolute_error(y_test, pipeline.predict(X_test))
score = pipeline.score(X_train, y_train)

print('Linear Regression Model')
print('Training MAE:', train_mae)
print('Validation MAE:', test_mae)
print('R^2 Score:', score)<jupyter_output>Linear Regression Model
Training MAE: 0.34579558840364066
Validation MAE: 2121766.133196058
R^2 Score: 0.5910491227649668<jupyter_text>Gradient Boosting<jupyter_code># define model
model = GradientBoostingRegressor()
# define transform
transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore'), cat),
                                             ('num', StandardScaler(), num)], remainder='passthrough')
# define pipeline
pipeline = Pipeline(steps=[('t', transformer), ('m',model)])
# fit the pipeline on the transformed data
pipeline.fit(X_train, y_train)
# make predictions
yhat = pipeline.predict(X_test)
g_train_mae = mean_absolute_error(y_train, pipeline.predict(X_train))
g_test_mae = mean_absolute_error(y_test, pipeline.predict(X_test))
g_score = pipeline.score(X_train, y_train)

print('Linear Regression Model')
print('Training MAE:', g_train_mae)
print('Validation MAE:', g_test_mae)
print('R^2 Score:', g_score)<jupyter_output>Linear Regression Model
Training MAE: 0.33190747927388253
Validation MAE: 0.33467449801991134
R^2 Score: 0.6231389218388331<jupyter_text>Random Forrest Regressor<jupyter_code># define model
model = RandomForestRegressor(n_estimators=200,
                              max_depth=20,
)

# define transform
transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore'), cat),
                                             ('num', StandardScaler(), num)], remainder='passthrough')
# define pipeline
pipeline = Pipeline(steps=[('t', transformer), ('m',model)])
# fit the pipeline on the transformed data
pipeline.fit(X_train, y_train)
# make predictions
yhat = pipeline.predict(X_test)
r_train_mae = mean_absolute_error(y_train, pipeline.predict(X_train))
r_test_mae = mean_absolute_error(y_test, pipeline.predict(X_test))
r_score = pipeline.score(X_train, y_train)
r_score_t = pipeline.score(X_test, y_test)

print('Random Forrest Regression Model')
print('Training MAE:', r_train_mae)
print('Validation MAE:', r_test_mae)
print('R^2 Score:', r_score)
print('R^2 t Score:', r_score_t)
from sklearn import metrics
from sklearn.model_selection import cross_val_score
scores = cross_val_score(pipeline, X_train, y_train, cv=5, scoring='f1_macro')
scores
print("%0.2f accuracy with a standard deviation of %0.2f" % (scores.mean(), scores.std()))
model_rr = make_pipeline(
    OneHotEncoder()
    RandomForestRegressor(),
)

model_lr.fit(X_train, y_train);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11621515508441303, 'fraction_numerical': 0.023360816647035727, 'mean_word_length': 5.3089783281733745, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '521806', 'license': 'MIT', 'n_tokens_mistral': 3426, 'n_tokens_neox': 3221, 'n_words': 876, 'path': 'Data_Exploration.ipynb', 'repo_name': 'StephenSpicer/airbnb_app', 'size': '10188'}