<jupyter_start><jupyter_text>NYC Rent**GOAL:** Improve our model for predicting NYC rent prices.  Wrangle Our Data<jupyter_code>%%capture
import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Applied-Modeling/master/data/'
    !pip install category_encoders==2.*

# If you're working locally:
else:
    DATA_PATH = '../data/'

!pip install category_encoders==2.*
import numpy as np
import pandas as pd

# Read New York City apartment rental listing data
df = pd.read_csv(DATA_PATH+'apartments/renthop-nyc.csv', 
                 parse_dates=['created'],
                 index_col='created')

df = df[(df['price'] >= np.percentile(df['price'], 0.5)) & 
        (df['price'] <= np.percentile(df['price'], 99.5)) & 
        (df['latitude'] >= np.percentile(df['latitude'], 0.05)) & 
        (df['latitude'] < np.percentile(df['latitude'], 99.95)) &
        (df['longitude'] >= np.percentile(df['longitude'], 0.05)) & 
        (df['longitude'] <= np.percentile(df['longitude'], 99.95))]
df.info()
df.describe()
df.cats_allowed.plot(kind='hist')
df.select_dtypes('object').head()
df.description.value_counts()<jupyter_output><empty_output><jupyter_text>Split Data<jupyter_code># Split FM and TV
y = df.price
X = df.drop(['price', 'description', 'display_address', 'street_address'], axis=1)
# Use data from April & May 2016 to train
# Use data from June 2016 to test
cutoff = '2016-06-01'
mask = X.index < cutoff

X_train, y_train = X.loc[mask], y.loc[mask]
X_val, y_val = X.loc[~mask], y.loc[~mask]
# Sanity check
assert X_train.shape[0] + X_val.shape[0] == X.shape[0]<jupyter_output><empty_output><jupyter_text>Establish Baseline<jupyter_code>y_train.plot(kind='hist')
from sklearn.metrics import mean_absolute_error

print('Baseline MAE:', mean_absolute_error(y_train, [y_train.mean()]*len(y_train)))<jupyter_output>Baseline MAE: 1201.8811133682555<jupyter_text>Build Model  Transformer 1: One hot encoding<jupyter_code>from category_encoders import OneHotEncoder
from sklearn.linear_model import LinearRegression
# Instantiate Transformer
ohe = OneHotEncoder(cols=['interest_level'], use_cat_names=True)

# Transform Feature Matrix
XT_train = ohe.fit_transform(X_train)

# Instantiate Predictor
model = LinearRegression()

# Fit on the TRANSFORMED data
model.fit(XT_train, y_train)
y_pred_train = model.predict(XT_train)

# Note that we DON'T retrain the transformer
XT_val = ohe.transform(X_val)
y_pred_val = model.predict(XT_val)
print('Training MAE:', mean_absolute_error(y_train, y_pred_train))
print('Validation MAE:', mean_absolute_error(y_val, y_pred_val))
model.score(XT_val, y_val)<jupyter_output><empty_output><jupyter_text>Transformer 2: SelectKBest<jupyter_code>len(X_train.columns)
from sklearn.feature_selection import SelectKBest

k_values = list(range(1,len(X_train.columns)+1))
r2_scores = []

for k in k_values:
  # Instantiate Transformer 1
  ohe = OneHotEncoder(cols=['interest_level'], use_cat_names=True)

  # Fit_transform 1
  XT_train = ohe.fit_transform(X_train)

  # Instantiate Transformer 2
  skb = SelectKBest(k=k)

  # Fit_transform 2
  XTT_train = skb.fit_transform(XT_train, y_train)

  # Instantiate Predictor
  model = LinearRegression()

  # Fit on the DOUBLE-TRANSFORMED data
  model.fit(XTT_train, y_train)

  # Get R^2 score
  r2_scores.append(model.score(XTT_train, y_train))
import matplotlib.pyplot as plt
plt.plot(k_values, r2_scores)
plt.xlabel('k (number of features selected)')
plt.ylabel('$R^2$ score');<jupyter_output><empty_output><jupyter_text>Predictor: Ridge<jupyter_code>from sklearn.linear_model import Ridge
model2 = Ridge()
model2.fit(XT_train, y_train)
model2.score(XT_val, y_val)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1361214704315397, 'fraction_numerical': 0.019978689397975494, 'mean_word_length': 5.802536231884058, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '398508', 'license': 'MIT', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1355, 'n_words': 269, 'path': '213_guided_project_notes.ipynb', 'repo_name': 'navroz-lamba/DS-Unit-2-Linear-Models', 'size': '3754'}