<jupyter_start><jupyter_text>IMPORTS<jupyter_code>import os
import pandas as pd
import numpy as np
import scipy.stats as stats
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing.data import QuantileTransformer
from sklearn import model_selection
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import RidgeCV
from sklearn.linear_model import LassoCV
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.tree import DecisionTreeRegressor
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error as mse
from sklearn.model_selection import cross_val_score
from sklearn.svm import SVC
from sklearn.datasets import make_classification
from yellowbrick.features import RFECV
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
from yellowbrick.regressor import ResidualsPlot
from sklearn.model_selection import GridSearchCV
from yellowbrick.regressor import AlphaSelection
from yellowbrick.model_selection import ValidationCurve
from sklearn.feature_selection import RFE<jupyter_output><empty_output><jupyter_text>Load Data Loads Data from Pickle File<jupyter_code>listings = pd.read_pickle("listings.pkl")
X = pd.read_pickle("X.pkl")
y = pd.read_pickle("y.pkl")
X_test = pd.read_pickle("X_test.pkl")
X_train = pd.read_pickle("X_train.pkl")
y_test = pd.read_pickle("y_test.pkl")
y_train = pd.read_pickle("y_train.pkl")<jupyter_output><empty_output><jupyter_text>Compares Initial Models<jupyter_code>models = []
models.append(('Linear', LinearRegression()))
models.append(('Ridge', RidgeCV()))
models.append(('Lasso', LassoCV(cv=5)))
models.append(('DecisionTree', DecisionTreeRegressor()))

scoring = 'r2'

results = []
names = []

res_scores = []
m_names = []
for name, model in models:
    print(name)
    pipe_mdl = make_pipeline(QuantileTransformer(),
                        model)
    kfold = model_selection.KFold(n_splits=12, random_state=42)
    scores = model_selection.cross_val_score(estimator=pipe_mdl,
                         X=X_train,
                         y=y_train,
                         cv=kfold,
                         scoring = scoring)
    print('CV accuracy: %.5f +/- %.5f' % (np.mean(scores),np.std(scores)))
    res_scores.append(scores)
    m_names.append(name)

fig = plt.figure(figsize=(10, 30))
fig.suptitle('Model Comparison')
ax = fig.add_subplot(111)
plt.boxplot(res_scores)
ax.set_xticklabels(m_names)
plt.show()
for name, model in models:
    visualizer = ResidualsPlot(model)
    visualizer.fit(X_train, y_train)
    visualizer.score(X_test, y_test)
    visualizer.poof()<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code># Creates feature ranking visualizer to find ideal number of features for each model

for name, model in models:
    print(name)
    viz = RFECV(model, n_jobs=-1)
    viz.fit(X, y)
    viz.poof()
# Uses recursive feature elimination to find features to drop/include based on the ideal number of features from the above model

print('Linear')
rfe = RFE(LinearRegression(), 40)
fit = rfe.fit(X, y)
print("Num Features: ", fit.n_features_) 
print("Selected Features: ", fit.support_) 
print("Feature Ranking: ", fit.ranking_)

print('Ridge')
rfe = RFE(RidgeCV(), 31)
fit = rfe.fit(X, y)
print("Num Features: ", fit.n_features_) 
print("Selected Features: ", fit.support_) 
print("Feature Ranking: ", fit.ranking_)

print('Lasso')
rfe = RFE(LassoCV(), 16)
fit = rfe.fit(X, y)
print("Num Features: ", fit.n_features_) 
print("Selected Features: ", fit.support_) 
print("Feature Ranking: ", fit.ranking_)

print('DecisionTree')
rfe = RFE(DecisionTreeRegressor(), 2)
fit = rfe.fit(X, y)
print("Num Features: ", fit.n_features_) 
print("Selected Features: ", fit.support_) 
print("Feature Ranking: ", fit.ranking_)
# Recreates X_train and X_test with final features

X_train_linear = X_train[[
    'year_is',
    'miles_fs',
    'seller_comments_length',
    'options_length',
    'features_length',
    'dom_is',
    'region_code',
    'region_id',
    #'agi_stub',
    'n1',
    #'mars1',
    'mars2',
    'n2',
    'numdep',
    'ral',
    'rac',
    'elderly',
    #'a00100',
    #'a02650',
    'n26270',
    'a03220',
    'n03300',
    'n03210',
    'n03230',
    'schf',
    'n18800',
    'n19300',
    'n19700',
    'model_ss_encoded',
    #'make_ss_encoded',
    'body_type_ss_encoded',
    'vehicle_type_ss_encoded',
    'fuel_type_ss_encoded',
    'engine_size_ss_encoded',
    'transmission_ss_encoded',
    'doors_is_encoded',
    'cylinders_is_encoded',
    'interior_color_ss_encoded',
    'exterior_color_ss_encoded',
    'state_ss_encoded',
    'county_encoded',
    'fips_code_encoded',
    'msa_type_encoded',
    'msa_name_encoded',
    'cbsa_encoded'
]]

X_test_linear = X_test[[
    'year_is',
    'miles_fs',
    'seller_comments_length',
    'options_length',
    'features_length',
    'dom_is',
    'region_code',
    'region_id',
    #'agi_stub',
    'n1',
    #'mars1',
    'mars2',
    'n2',
    'numdep',
    'ral',
    'rac',
    'elderly',
    #'a00100',
    #'a02650',
    'n26270',
    'a03220',
    'n03300',
    'n03210',
    'n03230',
    'schf',
    'n18800',
    'n19300',
    'n19700',
    'model_ss_encoded',
    #'make_ss_encoded',
    'body_type_ss_encoded',
    'vehicle_type_ss_encoded',
    'fuel_type_ss_encoded',
    'engine_size_ss_encoded',
    'transmission_ss_encoded',
    'doors_is_encoded',
    'cylinders_is_encoded',
    'interior_color_ss_encoded',
    'exterior_color_ss_encoded',
    'state_ss_encoded',
    'county_encoded',
    'fips_code_encoded',
    'msa_type_encoded',
    'msa_name_encoded',
    'cbsa_encoded'
]]

X_train_ridge = X_train[[
    'year_is',
    #'miles_fs',
    #'seller_comments_length',
    'options_length',
    'features_length',
    'dom_is',
    'region_code',
    'region_id',
    #'agi_stub',
    'n1',
    #'mars1',
    #'mars2',
    'n2',
    #'numdep',
    'ral',
    #'rac',
    'elderly',
    #'a00100',
    #'a02650',
    'n26270',
    'a03220',
    'n03300',
    'n03210',
    'n03230',
    'schf',
    #'n18800',
    #'n19300',
    #'n19700',
    'model_ss_encoded',
    #'make_ss_encoded',
    'body_type_ss_encoded',
    'vehicle_type_ss_encoded',
    'fuel_type_ss_encoded',
    'engine_size_ss_encoded',
    'transmission_ss_encoded',
    'doors_is_encoded',
    'cylinders_is_encoded',
    'interior_color_ss_encoded',
    'exterior_color_ss_encoded',
    'state_ss_encoded',
    #'county_encoded',
    'fips_code_encoded',
    'msa_type_encoded',
    'msa_name_encoded',
    'cbsa_encoded'
]]

X_test_ridge = X_test[[
    'year_is',
    #'miles_fs',
    #'seller_comments_length',
    'options_length',
    'features_length',
    'dom_is',
    'region_code',
    'region_id',
    #'agi_stub',
    'n1',
    #'mars1',
    #'mars2',
    'n2',
    #'numdep',
    'ral',
    #'rac',
    'elderly',
    #'a00100',
    #'a02650',
    'n26270',
    'a03220',
    'n03300',
    'n03210',
    'n03230',
    'schf',
    #'n18800',
    #'n19300',
    #'n19700',
    'model_ss_encoded',
    #'make_ss_encoded',
    'body_type_ss_encoded',
    'vehicle_type_ss_encoded',
    'fuel_type_ss_encoded',
    'engine_size_ss_encoded',
    'transmission_ss_encoded',
    'doors_is_encoded',
    'cylinders_is_encoded',
    'interior_color_ss_encoded',
    'exterior_color_ss_encoded',
    'state_ss_encoded',
    #'county_encoded',
    'fips_code_encoded',
    'msa_type_encoded',
    'msa_name_encoded',
    'cbsa_encoded'
]]

X_train_lasso = X_train[[
    #'year_is',
    'miles_fs',
    'seller_comments_length',
    'options_length',
    'features_length',
    'dom_is',
    'region_code',
    'region_id',
    'agi_stub',
    #'n1',
    #'mars1',
    #'mars2',
    'n2',
    #'numdep',
    #'ral',
    #'rac',
    #'elderly',
    'a00100',
    #'a02650',
    #'n26270',
    #'a03220',
    #'n03300',
    #'n03210',
    #'n03230',
    #'schf',
    'n18800',
    'n19300',
    'n19700',
    'model_ss_encoded',
    'make_ss_encoded',
    'body_type_ss_encoded',
    #'vehicle_type_ss_encoded',
    #'fuel_type_ss_encoded',
    #'engine_size_ss_encoded',
    #'transmission_ss_encoded',
    #'doors_is_encoded',
    #'cylinders_is_encoded',
    #'interior_color_ss_encoded',
    #'exterior_color_ss_encoded',
    #'state_ss_encoded',
    #'county_encoded',
    #'fips_code_encoded',
    #'msa_type_encoded',
    #'msa_name_encoded',
    #'cbsa_encoded'
]]

X_test_lasso = X_test[[
    #'year_is',
    'miles_fs',
    'seller_comments_length',
    'options_length',
    'features_length',
    'dom_is',
    'region_code',
    'region_id',
    'agi_stub',
    #'n1',
    #'mars1',
    #'mars2',
    'n2',
    #'numdep',
    #'ral',
    #'rac',
    #'elderly',
    'a00100',
    #'a02650',
    #'n26270',
    #'a03220',
    #'n03300',
    #'n03210',
    #'n03230',
    #'schf',
    'n18800',
    'n19300',
    'n19700',
    'model_ss_encoded',
    'make_ss_encoded',
    'body_type_ss_encoded',
    #'vehicle_type_ss_encoded',
    #'fuel_type_ss_encoded',
    #'engine_size_ss_encoded',
    #'transmission_ss_encoded',
    #'doors_is_encoded',
    #'cylinders_is_encoded',
    #'interior_color_ss_encoded',
    #'exterior_color_ss_encoded',
    #'state_ss_encoded',
    #'county_encoded',
    #'fips_code_encoded',
    #'msa_type_encoded',
    #'msa_name_encoded',
    #'cbsa_encoded'
]]


X_train_decisiontree = X_train[[
    'year_is',
    #'miles_fs',
    #'seller_comments_length',
    #'options_length',
    #'features_length',
    #'dom_is',
    #'region_code',
    #'region_id',
    #'agi_stub',
    #'n1',
    #'mars1',
    #'mars2',
    #'n2',
    #'numdep',
    #'ral',
    #'rac',
    #'elderly',
    #'a00100',
    #'a02650',
    #'n26270',
    #'a03220',
    #'n03300',
    #'n03210',
    #'n03230',
    #'schf',
    #'n18800',
    #'n19300',
    #'n19700',
    'model_ss_encoded',
    #'make_ss_encoded',
    #'body_type_ss_encoded',
    #'vehicle_type_ss_encoded',
    #'fuel_type_ss_encoded',
    #'engine_size_ss_encoded',
    #'transmission_ss_encoded',
    #'doors_is_encoded',
    #'cylinders_is_encoded',
    #'interior_color_ss_encoded',
    #'exterior_color_ss_encoded',
    #'state_ss_encoded',
    #'county_encoded',
    #'fips_code_encoded',
    #'msa_type_encoded',
    #'msa_name_encoded',
    #'cbsa_encoded'
]]

X_test_decisiontree = X_test[[
    'year_is',
    #'miles_fs',
    #'seller_comments_length',
    #'options_length',
    #'features_length',
    #'dom_is',
    #'region_code',
    #'region_id',
    #'agi_stub',
    #'n1',
    #'mars1',
    #'mars2',
    #'n2',
    #'numdep',
    #'ral',
    #'rac',
    #'elderly',
    #'a00100',
    #'a02650',
    #'n26270',
    #'a03220',
    #'n03300',
    #'n03210',
    #'n03230',
    #'schf',
    #'n18800',
    #'n19300',
    #'n19700',
    'model_ss_encoded',
    #'make_ss_encoded',
    #'body_type_ss_encoded',
    #'vehicle_type_ss_encoded',
    #'fuel_type_ss_encoded',
    #'engine_size_ss_encoded',
    #'transmission_ss_encoded',
    #'doors_is_encoded',
    #'cylinders_is_encoded',
    #'interior_color_ss_encoded',
    #'exterior_color_ss_encoded',
    #'state_ss_encoded',
    #'county_encoded',
    #'fips_code_encoded',
    #'msa_type_encoded',
    #'msa_name_encoded',
    #'cbsa_encoded'
]]<jupyter_output><empty_output><jupyter_text>Hyperparameter Tuning<jupyter_code># Creates Alphas
alphas = np.logspace(-10, 1, 1000)

# Instantiate the linear model and visualizer
model = RidgeCV(normalize = True)
visualizer = AlphaSelection(model)

visualizer.fit(X_train_ridge, y_train)
g = visualizer.poof()

# Instantiate the Lasso model and visualizer
model = LassoCV(alphas=alphas, normalize = True)
visualizer = AlphaSelection(model)

visualizer.fit(X_train_lasso, y_train)
g = visualizer.poof()
# Instantiate the Decision Tree model and visualizer

viz = ValidationCurve(
    DecisionTreeRegressor(), param_name="max_depth",
    param_range=np.arange(1, 15), cv=10, scoring="r2"
)

# Fit and poof the visualizer
viz.fit(X_train_decisiontree, y_train)
viz.poof()<jupyter_output><empty_output><jupyter_text>Run New Models with Final Hyperparameters and Features<jupyter_code>models = []
models.append(('Linear', LinearRegression()))
models.append(('Ridge', RidgeCV()))
models.append(('Lasso', LassoCV(cv=5)))
models.append(('DecisionTree', DecisionTreeRegressor()))

#Linear Regression Model
print('Linear Model')
regr = LinearRegression()

# Train the model using the training sets
regr.fit(X_train_linear, y_train)

# Make predictions using the testing set
y_pred = regr.predict(X_test_linear)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print("Mean squared error: %.2f"
      % mse(y_test, y_pred))
# Explained variance score: 1 is perfect prediction
print('R2: %.2f' % r2_score(y_test, y_pred))

#Ridge Model
print('Ridge')
regr = Ridge(alpha = .1)

# Train the model using the training sets
regr.fit(X_train_ridge, y_train)

# Make predictions using the testing set
y_pred = regr.predict(X_test_ridge)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print("Mean squared error: %.2f"
      % mse(y_test, y_pred))
# Explained variance score: 1 is perfect prediction
print('R2: %.2f' % r2_score(y_test, y_pred))

#Lasso Model
print('Lasso')
regr = Lasso(alpha = .004)

# Train the model using the training sets
regr.fit(X_train_lasso, y_train)

# Make predictions using the testing set
y_pred = regr.predict(X_test_lasso)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print("Mean squared error: %.2f"
      % mse(y_test, y_pred))
# Explained variance score: 1 is perfect prediction
print('R2: %.2f' % r2_score(y_test, y_pred))

#Decision Tree Model
print('Decision Tree')
regr = DecisionTreeRegressor(max_depth = 8)

# Train the model using the training sets
regr.fit(X_train_decisiontree, y_train)

# Make predictions using the testing set
y_pred = regr.predict(X_test_decisiontree)

# The mean squared error
print("Mean squared error: %.2f"
      % mse(y_test, y_pred))
# Explained variance score: 1 is perfect prediction
print('R2: %.2f' % r2_score(y_test, y_pred))
#Linear Model
visualizer = ResidualsPlot(LinearRegression())
visualizer.fit(X_train_linear, y_train)
visualizer.score(X_test_linear, y_test)
visualizer.poof()

#Ridge Model
visualizer = ResidualsPlot(Ridge(alpha = .1))
visualizer.fit(X_train_ridge, y_train)
visualizer.score(X_test_ridge, y_test)
visualizer.poof()

#Lasso Model
visualizer = ResidualsPlot(Lasso(alpha = .004))
visualizer.fit(X_train_lasso, y_train)
visualizer.score(X_test_lasso, y_test)
visualizer.poof()

#Decision Tree Model
visualizer = ResidualsPlot(DecisionTreeRegressor(max_depth = 8))
visualizer.fit(X_train_decisiontree, y_train)
visualizer.score(X_test_decisiontree, y_test)
visualizer.poof()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1426667555140934, 'fraction_numerical': 0.03331778503365096, 'mean_word_length': 4.054900639946109, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '176376', 'license': 'MIT', 'n_tokens_mistral': 6185, 'n_tokens_neox': 5624, 'n_words': 1003, 'path': 'scripts/Modeling_Notebook.ipynb', 'repo_name': 'georgetown-analytics/Used-Cars-Analyzers', 'size': '15007'}