<jupyter_start><jupyter_text>Why & how were target, metric, and baseline chosen?How was leakage avoided. When & why is the model (or not) useful?Price was chosen as a natural target because it is information that would be most applicable to the user. Whether someone is looking to sell or buy a car, price is the bottom line.Columns did not contain leakage because the price of the post was declared at the time of making the post. The features included are not related to a sale price.Started with a baseline of 8613.48 MAE of a mean baseline or a 7829.12 MAE of the simplest linear regression based on ‘year’. Quickly beat these scores with a decision tree. For metrics, I chose R^2 for variance explainability and MAE for simple error comparison.Limitations: The model is mostly useful for a quick price estimation on used cars. Getting a specific model or sale price can be done on the larger original used cars dataset.<jupyter_code># Choose target
target = 'price'
# Is your problem regression or classification?
# Regression, predicting a continuous variable with many levels
# Target distribution
fig, ax = plt.subplots()
sns.distplot(df1['price'])
ax.set_ylabel('proportion')
ax.set_title('Price Distribution')
pd.options.display.float_format = '{:.1f}'.format
print(np.median(df1['price']))
y = df1['price']
y.describe()
# Very right skewed but the 0s are holding it back even more. 
# Let's look at logs
y = y.replace(0, np.NaN)
sns.distplot(y)
# It was deemed not worth log transforming, I did some modeling below and R^2 was worse
# y_log = np.log1p(y)
# sns.distplot(y_log)
# Chop out some outliers

df2 = df1
df2['price'] = df2['price'].replace(0,np.NaN)
df2 = df2.dropna(subset=['price'])
df2 = df2[(df2['price'] >= np.percentile(df2['price'], 0.05)) &
          (df2['price'] <= np.percentile(df2['price'], 99.95))]
df2['price'].describe()
fig, ax = plt.subplots()
sns.distplot(df2['price'])
ax.set_ylabel('proportion')
ax.set_title('Price Distribution')
y = np.log1p(df2['price'])
sns.distplot(y)
# Now it's left skewed. Perhaps not worth log transform
y.describe()
# Dollar amount of the average
np.e**9.1
# Choose your evaluation metric(s).

#  I chose R^2 for variance explainability and MAE for simple error comparison.
#Choose which observations you will use to train, validate, and test your model.
train_orig, test = train_test_split(df2.drop(columns=['county','Unnamed: 0']) , test_size = 0.1, train_size = 0.9)
test
train, val = train_test_split(train_orig, train_size = 0.9, test_size = 0.1)
train
# MEAN BASELINE
#  For the evaluation metric you chose, what score would you get just by guessing?
# R^2 for a flat line of the mean would be 0. Let's look at the MAE
list_of_mean = [train['price'].mean()] * len(val['price'])
mean_absolute_error(val['price'], list_of_mean)
# LINEAR REGRESSION (to beat) BASELINE
# Let's do a univariate linear regression as a baseline as well
imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
imp_mean.fit(train[['year']])
lr = LinearRegression()
model = lr.fit(imp_mean.transform(train[['year']]),train['price'])

print('MAE:', mean_absolute_error(val['price'], model.predict(imp_mean.transform(val[['year']]))))
print('R^2:', r2_score(val['price'], model.predict(imp_mean.transform(val[['year']]))))
# Alrighty then. Got some baselines!
target = 'price'
features = ['region', 'year', 'manufacturer', 'model', 'condition',
       'cylinders', 'fuel', 'odometer', 'title_status', 'transmission',
       'drive', 'size', 'type', 'paint_color', 'state', 'lat',
       'long']
X_train = train[features]
y_train = train[target]
X_val = val[features]
y_val = val[target]
X_test = test[features]
y_test = test[target]
# Heck let's do a decision tree as a baseline too
pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='median'),
    DecisionTreeRegressor(max_depth=10)
)
pipeline.fit(X_train, y_train)

y_pred = pipeline.predict(X_val)
print('MAE:', mean_absolute_error(y_val, y_pred))
print('R^2:', r2_score(y_val, y_pred))

# Pretty dope tree
# Explore more
train.dtypes
sns.scatterplot(train['year'], train['price'])
plt.title('price by year')
train['manufacturer'].value_counts().plot(kind='bar')
sns.scatterplot(train['paint_color'], train['price'])
plt.title('price by color')
# BEAT BASELINE WITH TREES
%time
# Random Forest model

pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='median'),
    RandomForestRegressor(n_estimators=50)
)
pipeline.fit(X_train, y_train)

y_pred = pipeline.predict(X_val)
print('MAE:', mean_absolute_error(y_val, y_pred))
print('R^2:', r2_score(y_val, y_pred))

# lit, already happy about that
# Earlier models that needed extra tuning
# XGBoost

# pipeline = make_pipeline(
#     ce.OrdinalEncoder(), 
#     # SimpleImputer(strategy='median'),  missing values accepted in this
#     XGBRegressor(n_estimators=70, random_state=42, n_jobs=-1)
# )
# pipeline.fit(X_train, y_train)

# y_pred = pipeline.predict(X_val)
# print('MAE:', mean_absolute_error(y_val, y_pred))
# print('R^2:', r2_score(y_val, y_pred))

# Much to tune
# Early stopping, 
# encoder = ce.OrdinalEncoder()
# X_train_encoded = encoder.fit_transform(X_train)
# X_val_encoded = encoder.transform(X_val)
# model = XGBRegressor(
#     n_estimators= 500, # up to 1000, depends on early stop
#     max_depth = 7,
#     learning_rate= 0.5, # try higher learning rate
# )

# eval_set = [(X_train_encoded, y_train), (X_val_encoded, y_val)]

# model.fit(X_train_encoded,y_train,
#           eval_set = eval_set,
#           eval_metric = 'mae',
#           early_stopping_rounds = 50)
# Random forest still better, will have to play with it
# Early stopping, TUNED
%time
encoder = ce.OrdinalEncoder()
X_train_encoded = encoder.fit_transform(X_train)
X_val_encoded = encoder.transform(X_val)
model = XGBRegressor(
    n_estimators= 100, # up to 100, depends on early stop
    max_depth = 35,
    learning_rate= 0.10, # Lower rate helps with overfitting.
    n_jobs=-3
)

eval_set = [(X_train_encoded, y_train), (X_val_encoded, y_val)]

model.fit(X_train_encoded,y_train,
          eval_set = eval_set, # This is what is being scored (why we have two col)
          eval_metric = 'mae',
          early_stopping_rounds = 2) #stop if score not improved in n rounds
    
# Let's plot XGBoost's journey

results = model.evals_result_
training_err = results['validation_0']['mae']
validation_err = results['validation_1']['mae']
epoch = list(range(0,len(training_err)))

plt.plot(epoch,training_err, label = 'Training Error')
plt.plot(epoch,validation_err, label = 'Validation Error')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Mean Average Error')
plt.title('XGBoost MAE by Round')
# Can we just take the output model and use that??
%time
encoder = ce.OrdinalEncoder()
X_train_encoded = encoder.fit_transform(X_train)
X_val_encoded = encoder.transform(X_val)

model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
             colsample_bynode=1, colsample_bytree=1, gamma=0,
             importance_type='gain', learning_rate=0.1, max_delta_step=0,
             max_depth=35, min_child_weight=1, missing=None, n_estimators=350,
             n_jobs=-3, nthread=None, objective='reg:linear', random_state=0,
             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
             silent=None, subsample=1, verbosity=1)
model.fit(X_train_encoded, y_train)
# XGBoost R^2
y_pred = model.predict(X_val_encoded)
print('R^2:', r2_score(y_val, y_pred))

# BEAT BASELINE, slightly better than RF R^2= 0.8407 (scores vary with iteration)
# Clean up for permutation importances/ pdp

imputer = SimpleImputer(strategy='median')
X_val_imputed = imputer.fit_transform(X_val_encoded)
#df_val_imputed = pd.DataFrame(data= val_imputed, columns=val.columns)
# X_val_imputed = df_val_imputed[features]


xgb = model.fit(X_val_imputed, y_val)
# Permutation importances
permuter = PermutationImportance(
    model,
    scoring= 'r2',
    n_iter = 5,
    random_state=42

)

permuter.fit(X_val_imputed, y_val)
feature_names = X_val.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values(ascending=False)

# Here I can consider cutting down columns, would it make a better app?
# predictively I'm ok, not great, so should I prioritize user experience
eli5.show_weights(
    permuter,
    top=11, # number of features to show, no limit
    feature_names= feature_names, # must be a list
)
# x Make at least 1 partial dependence plot to explain your model.

val_imp_df = pd.DataFrame(X_val_imputed, columns = X_val.columns)
feature = 'year'

isolated = pdp_isolate(
    model = pipeline,
    dataset= val_imp_df,
    model_features= X_val.columns,
    feature = feature
)
pdp_plot(isolated, feature_name= feature)

# neat. You can kinda see what cars people put up for sale have value.
# let's look at that old year
val.sort_values(by='year').head(4)
# STRETCH
#  Make multiple PDPs with 2 features in interaction.

feature= ['year', 'odometer']

interaction = pdp_interact(
    model = pipeline,
    dataset = val_imp_df,
    model_features= X_val.columns,
    features= feature
)
pdp_interact_plot(interaction, plot_type='grid', feature_names=feature)
# Kinda meaningless without the drive dictionary
#  Make at least 1 Shapley force plot to explain an individual prediction.

row = X_val_encoded.iloc[5]
y_val.iloc[[5]], model.predict(row)
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(row)

shap.initjs()
shap.force_plot(
    base_value= explainer.expected_value,
    shap_values = shap_values,
    features= row

)

## Final model for app usage

target = 'price'
features1 = ['year', 'manufacturer', 'cylinders','fuel', 'odometer', 'drive']
X_train1 = train[features1]
y_train = train[target]
X_val1 = val[features1]
y_val = val[target]
X_test1 = test[features1]
y_test = test[target]
# pipeline1 = make_pipeline(
#     ce.TargetEncoder(),
#     SimpleImputer(strategy='median'),
#     RandomForestRegressor(bootstrap=True, ccp_alpha=0.0,
#                                        criterion='mae', max_depth=18,
#                                        max_features='auto', max_leaf_nodes=None,
#                                        max_samples=None,
#                                        min_impurity_decrease=0.0,
#                                        min_impurity_split=None,
#                                        min_samples_leaf=1, min_samples_split=2,
#                                        min_weight_fraction_leaf=0.0,
#                                        n_estimators=35, n_jobs=None,
#                                        oob_score=False, random_state=42,
#                                        verbose=0, warm_start=False)
# )
# pipeline1.fit(X_train1, y_train)

# y_pred = pipeline1.predict(X_val1)
# print('MAE:', mean_absolute_error(y_val, y_pred))
# print('R^2:', r2_score(y_val, y_pred))


# pickle the chosen model
# dump(pipeline1, 'pipeline_drive.joblib', compress=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11920950049859487, 'fraction_numerical': 0.016226996645816335, 'mean_word_length': 4.263358778625954, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '99426', 'license': 'MIT', 'n_tokens_mistral': 3822, 'n_tokens_neox': 3661, 'n_words': 976, 'path': 'notebooks/Used_cars_Build.ipynb', 'repo_name': 'JonNData/used-car-predictor', 'size': '11031'}