<jupyter_start><jupyter_text><jupyter_code>%%capture
import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Applied-Modeling/master/data/'
    !pip install category_encoders==2.*

# If you're working locally:
else:
    DATA_PATH = '../data/'
    
# Ignore this Numpy warning when using Plotly Express:
# FutureWarning: Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')
import numpy as np
import pandas as pd

# Read New York City apartment rental listing data
df = pd.read_csv(DATA_PATH+'apartments/renthop-nyc.csv', parse_dates=['created'])#, index_col='created')
assert df.shape == (49352, 34)

# Remove the most extreme 1% prices,
# the most extreme .1% latitudes, &
# the most extreme .1% longitudes
df = df[(df['price'] >= np.percentile(df['price'], 0.5)) & 
        (df['price'] <= np.percentile(df['price'], 99.5)) & 
        (df['latitude'] >= np.percentile(df['latitude'], 0.05)) & 
        (df['latitude'] < np.percentile(df['latitude'], 99.95)) &
        (df['longitude'] >= np.percentile(df['longitude'], 0.05)) & 
        (df['longitude'] <= np.percentile(df['longitude'], 99.95))]
df.columns
df.info()<jupyter_output><empty_output><jupyter_text>1. Engineer Features<jupyter_code>#df['apt_desc_yes_or_no'] = df['description'].apply(lambda x: 0 if df.loc[df['description']== np.NaN] else 1 )
#cond = df['description']== np.NaN
#df.loc[df['description']== np.NaN,'apt_desc_yes_or_no' ] = 0

# Adding description length
df['apt_desc_len'] = len(df['description'])

# All perks
df['perks'] = df.iloc[:, 9:-2].sum(axis = 1)

# dogs and cats
df['dogs and cats'] = df['dogs_allowed'] + df['cats_allowed']

df['dogs or cats'] = [1 if x == 1 else 0 for x in df['dogs and cats']]
df['dogs_and_cats'] = [1 if x == 2 else 0 for x in df['dogs and cats']]
df['pets'] = [0 if x == 0 else 1 for x in df['dogs and cats']]
df= df.drop('dogs and cats', axis= 1)

# Total rooms
df['total_rooms'] = df['bedrooms'] + df['bathrooms']

# Room ratio
df['bath_per_bed'] = df['bathrooms']/ df['bedrooms'] 

# Add month colum
df['month'] = df['created'].dt.month

df = df.replace({'interest_level':{"medium": 2, "low": 1, "high": 3}})


df.head(2)<jupyter_output><empty_output><jupyter_text>Train/Test Split<jupyter_code>df = df.dropna()
df.isnull().sum()
mask = df.month == 6
train = df.loc[mask]
test = df.loc[~mask]

train.shape, test.shape<jupyter_output><empty_output><jupyter_text>Split feature and target matrices<jupyter_code>target = 'price'
y_train = train[target]
X_train = train.drop(['price', 'created', 'description', 'display_address', 'street_address'], axis = 1)

y_test = test[target]
X_test = test.drop(['price', 'created', 'description', 'display_address', 'street_address'], axis = 1)


print(X_train.shape, y_train.shape)
print(X_test.shape, y_test.shape)<jupyter_output>(16402, 37) (16402,)
(30707, 37) (30707,)<jupyter_text>Baseline<jupyter_code>baseline = y_train.mean()
print(f'baseline price is: ${baseline}')
from sklearn.metrics import mean_absolute_error, mean_squared_error,r2_score
# baseline metrics

y_pred = [baseline] * len(y_train)
baseline_MAE = mean_absolute_error(y_train, y_pred)
baseline_R2 = r2_score(y_train, y_pred)
baseline_MSE = mean_squared_error(y_train, y_pred)
print(f'Baseline MAE: ${baseline_MAE}')
print(f'Baseline R2: ${baseline_R2}')
print(f'Baseline RMSE: ${baseline_MSE}')<jupyter_output>Baseline MAE: $1204.6323797195153
Baseline R2: $0.0
Baseline RMSE: $3123768.66377459<jupyter_text>Build a Model<jupyter_code>X_train.columns
from sklearn.linear_model import LinearRegression

model_all = LinearRegression()

model_all.fit(X_train.iloc[:, 0:-3], y_train)

pred_all = model_all.predict(X_test.iloc[:, 0:-3])
len(pred_all)
len(y_test)<jupyter_output><empty_output><jupyter_text>Metrics<jupyter_code># test metrics

#y_test_pred = [pred_all] * len(y_test)
test_MAE = mean_absolute_error(y_test, pred_all)
test_R2 = r2_score(y_test, pred_all)
test_MSE = mean_squared_error(y_test, pred_all)
print(f'Test MAE: ${test_MAE}')
print(f'Test R2: ${test_R2}')
print(f'Test RMSE: ${test_MSE}')
train_pred = model_all.predict(X_train.iloc[:, 0:-3])
train_R2 = r2_score(y_train, train_pred)
train_R2
train_MAE = mean_absolute_error(y_train, train_pred)
train_MAE<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16636690647482014, 'fraction_numerical': 0.029676258992805755, 'mean_word_length': 6.245928338762215, 'pattern_counts': {'":': 3, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '238399', 'license': 'MIT', 'n_tokens_mistral': 1812, 'n_tokens_neox': 1684, 'n_words': 355, 'path': 'module2-regression-2/2.2_Assignments_py.ipynb', 'repo_name': 'JenBanks8585/DS-Unit-2-Linear-Models', 'size': '4448'}