<jupyter_start><jupyter_text>Model  Importing Data<jupyter_code># Import necessary modules
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')

import seaborn as sns

# Display 100 columns max
pd.set_option('display.max_columns', 100)

# Display 20 characters max within a cell
# pd.set_option('max_colwidth',60)

from sklearn.model_selection import train_test_split
from sklearn.linear_model import Lasso
from sklearn.model_selection import cross_val_score

from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_val_score

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import RandomizedSearchCV

from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

import mlflow

import pickle
# Import imputed data
df = pd.read_pickle("autos_golden_20190719.pkl")

# Import data without missing values (No imputation, no missing in the data)
df_clean = pd.read_pickle("autos_clean_golden_20190719.pkl")

# Import data with missing values (No imputation)
df_missing = pd.read_pickle("autos_missing_golden_20190719.pkl")
df.shape, df_missing.shape
df_clean.shape
df.head()<jupyter_output><empty_output><jupyter_text>Model<jupyter_code># Create features and target (df)
X = df.drop('price', axis = 1) # features
y = df['price'] # target
# Create features and target (df)
X_c = df_clean.drop('price', axis = 1) # features
y_c = df_clean['price'] # target
# # Create features and target (array)
# X = df.drop('price', axis = 1).values # features
# y = df['price'].values # target
# Train-test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=57)
# Train-test split
X_train_c, X_test_c, y_train_c, y_test_c = train_test_split(X_c, y_c, test_size = 0.3, random_state=57)<jupyter_output><empty_output><jupyter_text>Model Evaluation Metrics<jupyter_code>def eval_metrics(actual, pred):
    rmse = np.sqrt(mean_squared_error(actual, pred))
    mae = mean_absolute_error(actual, pred)
    r2 = r2_score(actual, pred)
    return rmse, mae, r2<jupyter_output><empty_output><jupyter_text>Linear Regression<jupyter_code>from sklearn.preprocessing import PolynomialFeatures

poly = PolynomialFeatures(interaction_only=True,include_bias = False)
poly.fit_transform(X)
X.columns<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code># Create Alphas, and lists to store scores
alpha_space = np.linspace(1, 10)
lasso_scores = []

# Create a regressor
lasso_ = Lasso(normalize = True)

# Compute scores over range of alphas
for alpha in alpha_space:
    
    # Specify alpha
    lasso_.alpha = alpha
    
    # Perfrom 5-fold CV
    lasso_cv_scores = cross_val_score(lasso_, X_train, y_train, cv=5)
    
    # Append the mean and std to the lists
    lasso_scores.append(np.mean(lasso_cv_scores))
    
    # Log parameter, metrics, and model to MLflow
    with mlflow.start_run(run_name="Lasso CV"): # "with" is necessary for multiple runs
        mlflow.log_param("alpha", alpha)
        mlflow.log_metric("lasso_cv_score", np.mean(lasso_cv_scores))
#!mlflow ui
# Create a function to display how cv scores changes with alphas
def display_plot(cv_scores):
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    ax.plot(alpha_space, cv_scores)

    ax.set_ylabel('CV Score')
    ax.set_xlabel('Alpha')
    ax.axhline(np.max(cv_scores), linestyle='--', color='.5')
    ax.set_xlim([alpha_space[0], alpha_space[-1]])
    ax.set_xscale('log')
    ax.axvline(x=1.4, c='b', linestyle='--', alpha=0.5)
    plt.show()
    
display_plot(lasso_scores)<jupyter_output><empty_output><jupyter_text>$alpha = 1.4$ seems to be a good choice<jupyter_code># Instantiate Lasso regressor
lasso = Lasso(alpha = 1.4, normalize = True)

# Fit the regressor
lasso.fit(X_train, y_train)

# Print the R-squared
print("R^2: {0:.3f}".format(lasso.score(X_test, y_test)))

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test, lasso.predict(X_test))

# Log parameter, metrics, and model to MLflow
with mlflow.start_run(run_name="Lasso Regression"):
    mlflow.log_param("alpha", 1.4)
    mlflow.log_metric("rmse", rmse)
    mlflow.log_metric("mae", mae)
    mlflow.log_metric("r2", r2)
#mlflow.end_run()
# Perform 5-fold cross-validation
cv_scores = cross_val_score(lasso, X_test, y_test, cv=5)

print("Average 5-Fold CV Score: {0:.3f}".format(np.mean(cv_scores)))<jupyter_output>Average 5-Fold CV Score: 0.856<jupyter_text>Linear Regression<jupyter_code># Create the regressor
reg = LinearRegression()

# Fit the regressor to the training data
reg.fit(X_train, y_train)

# Print the R-squared
print("R^2: {0:.2f}".format(reg.score(X_test, y_test)))

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test, reg.predict(X_test))

# Log parameter, metrics, and model to MLflow
mlflow.start_run(run_name="Linear Regression")
mlflow.log_param("normalize", False)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("mae", mae)
mlflow.log_metric("r2", r2)
mlflow.end_run()
# Create the regressor
reg = LinearRegression(normalize=True)

# Fit the regressor to the training data
reg.fit(X_train, y_train)

# Print the R-squared
print("R^2: {0:.2f}".format(reg.score(X_test, y_test)))

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test, reg.predict(X_test))

# Log parameter, metrics, and model to MLflow
mlflow.start_run(run_name="Linear Regression")
mlflow.log_param("normalize", True)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("mae", mae)
mlflow.log_metric("r2", r2)
mlflow.end_run()
# Create the regressor
reg = LinearRegression()

# Fit the regressor to the training data
reg.fit(X_train_c, y_train_c)

# Print the R-squared
print("R^2: {0:.2f}".format(reg.score(X_test_c, y_test_c)))

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test, reg.predict(X_test))

# Log parameter, metrics, and model to MLflow
with mlflow.start_run(run_name="Linear Regression 'Dropped Missing'"):
    mlflow.log_metric("rmse", rmse)
    mlflow.log_metric("mae", mae)
    mlflow.log_metric("r2", r2)<jupyter_output>R^2: 0.89<jupyter_text>Random Forest<jupyter_code>preds_rf = reg_rf.predict(X_test)

rmse = np.sqrt(mean_squared_error(y_test, preds_rf))
print("RMSE: %f" % (rmse))
reg_rf = RandomForestRegressor(random_state=57, n_estimators=100)

reg_rf.fit(X_train_c, y_train_c)

reg_rf.score(X_test_c, y_test_c)

#pd.DataFrame({'x':X_hp.columns.values, 'y':reg_rf.feature_importances_})

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test_c, reg_rf.predict(X_test_c))

# Log parameter, metrics, and model to MLflow
with mlflow.start_run(run_name="Random Forest 'Dropped Missing'"):
    mlflow.log_metric("rmse", rmse)
    mlflow.log_metric("mae", mae)
    mlflow.log_metric("r2", r2)
preds_rf = reg_rf.predict(X_test_c)

rmse = np.sqrt(mean_squared_error(y_test_c, preds_rf))
print("RMSE: %f" % (rmse))
print("R2: ", r2)
feature_importance = pd.DataFrame({'features':X_c.columns.values, 'importance':reg_rf.feature_importances_}).sort_values('importance', ascending=False)
feature_importance.head(10).set_index('features')
feature_importance.set_index('features')[:10].plot.bar(rot=60)<jupyter_output><empty_output><jupyter_text>XGBoost with Imputed Missing Values<jupyter_code>import xgboost as xgb

xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
                max_depth = 5, alpha = 10, n_estimators = 100)

xg_reg.fit(X_train, y_train)

preds = xg_reg.predict(X_test)

rmse = np.sqrt(mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test, xg_reg.predict(X_test))

# Log parameter, metrics, and model to MLflow
mlflow.start_run(run_name="XGBoost Imputed")
mlflow.log_param("alpha", 10)
mlflow.log_param("max_depth", 5)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("mae", mae)
mlflow.log_metric("r2", r2)
mlflow.end_run()
xgb.plot_importance(xg_reg, max_num_features=20)<jupyter_output><empty_output><jupyter_text>XGBoost with Dropped Missing Values<jupyter_code>import xgboost as xgb

xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
                max_depth = 5, alpha = 10, n_estimators = 100)

xg_reg.fit(X_train_c, y_train_c)

preds = xg_reg.predict(X_test_c)

rmse = np.sqrt(mean_squared_error(y_test_c, preds))
print("RMSE: %f" % (rmse))

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test_c, xg_reg.predict(X_test_c))

# Log parameter, metrics, and model to MLflow
mlflow.start_run(run_name="XGBoost - 'Dropped Missing'")
mlflow.log_param("alpha", 10)
mlflow.log_param("max_depth", 5)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("mae", mae)
mlflow.log_metric("r2", r2)
mlflow.end_run()
xgb.plot_importance(xg_reg, max_num_features=20)<jupyter_output><empty_output><jupyter_text>XGBoost with Missing Values<jupyter_code># Create features and target (df)
X_m = df_missing.drop('price', axis = 1) # features
y_m = df_missing['price'] # target

# Train-test split
X_train_m, X_test_m, y_train_m, y_test_m = train_test_split(X_m, y_m, test_size = 0.3, random_state=57)
xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
                max_depth = 5, alpha = 10, n_estimators = 100)

xg_reg.fit(X_train_m, y_train_m)

preds = xg_reg.predict(X_test_m)

rmse = np.sqrt(mean_squared_error(y_test_m, preds))
print("RMSE: %f" % (rmse))

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test_m, xg_reg.predict(X_test_m))

# Log parameter, metrics, and model to MLflow
mlflow.start_run(run_name="XGBoost NOT Imputed")
mlflow.log_param("alpha", 10)
mlflow.log_param("max_depth", 5)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("mae", mae)
mlflow.log_metric("r2", r2)
mlflow.end_run()
xgb.plot_importance(xg_reg, max_num_features=20)<jupyter_output><empty_output><jupyter_text>Save Model (4 features)<jupyter_code># Import imputed data
df_no_dummy = pd.read_pickle("autos_clean_no_dummy_20190805.pkl")
test_data = df_no_dummy[['hp', 'age', 'km', 'model', 'price']].copy()
# Create features and target (df)
X = pd.get_dummies(df_no_dummy[['hp', 'age', 'km', 'model']]) # features
y = df_no_dummy['price'] # target
X.head()
y.head()
# Train-test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=57)
reg_rf = RandomForestRegressor(n_estimators=100, random_state=57)

reg_rf.fit(X_train, y_train)

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test, reg_rf.predict(X_test))

print("rmse: {}, mae: {}, r2: {}".format(rmse, mae, r2))
from sklearn.model_selection import RandomizedSearchCV

# Number of trees in random forest
n_estimators = [int(x) for x in np.linspace(start = 100, stop = 2000, num = 10)]
# Number of features to consider at every split
max_features = ['auto', 'sqrt']
# Maximum number of levels in tree
max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
max_depth.append(None)
# Minimum number of samples required to split a node
min_samples_split = [2, 5, 10]
# Minimum number of samples required at each leaf node
min_samples_leaf = [1, 2, 4]
# Method of selecting samples for training each tree
bootstrap = [True, False]

# Create the random grid
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}
# Use the random grid to search for best hyperparameters
# First create the base model to tune
rf = RandomForestRegressor()
# Random search of parameters, using 3 fold cross validation, 
# search across 100 different combinations, and use all available cores
rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 10, cv = 3, verbose=2, random_state=42, n_jobs = -1)
# Fit the random search model
rf_random.fit(X_train, y_train)
rf_random.best_params_
rf_best = rf_random.best_estimator_
# reg_rf = RandomForestRegressor(n_estimators=100, random_state=57)

# reg_rf.fit(X_train, y_train)

# Calculate evaluation metrics
rmse, mae, r2 = eval_metrics(y_test, rf_best.predict(X_test))

print("rmse: {}, mae: {}, r2: {}".format(rmse, mae, r2))
# Save RandomForest model
# pickle.dump(rf_best, open('model_rf.pkl','wb'))
list(X.columns)
# Save column names to persist the features in deployment phase
# pickle.dump(list(X.columns), open('model_features.pkl','wb'))
test_data['prediction'] = rf_best.predict(X)
test_data.sample(5)
# Create subplots
fig, ax = plt.subplots(2, 4, figsize=(15, 12))

# Plot subplots
i = 0
j = 0

for model in ['A1', 'A3', 'Astra', 'Clio', 'Corsa', 'Espace', 'Insignia']:
    
    df_no_dummy[df_no_dummy.model == model].boxplot('price', by='age', ax = ax[j, i])
    ax[j, i].set_title(model)
    # Next subplot
    i += 1
    
    if i == 4:
        i = 0
        j = 1

df_no_dummy.model.value_counts()
sns.pairplot(df_no_dummy[df_no_dummy.model=='Clio'][['age', 'price']], kind='reg')
rf_best.decision_path(X)
rf_best
rf_best.fit(X_train, y_train)
import graphviz
import pydot


tree = rf_best.estimators_[5]
# Export the image to a dot file
export_graphviz(tree, out_file = 'tree.dot', feature_names = X_train.columns, 
rounded = True, precision = 1)
# Use dot file to create a graph
(graph, ) = pydot.graph_from_dot_file('tree.dot')
# Write graph to a png file
graph.write_png('tree.png')
from statsmodels.graphics.factorplots import interaction_plot
fig = interaction_plot(df_no_dummy.age, df_no_dummy.model, df_no_dummy.price)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12696106914584543, 'fraction_numerical': 0.01910226612434631, 'mean_word_length': 5.802865612648222, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '490369', 'license': 'MIT', 'n_tokens_mistral': 5457, 'n_tokens_neox': 5194, 'n_words': 1135, 'path': 'trgt/model-final.ipynb', 'repo_name': 'kaymal/price-project', 'size': '13768'}