<jupyter_start><jupyter_text>UBC-MDS/DSCI_522_group_401 Predicting Medical Insurance CostAuthors: <NAME>, <NAME>, <NAME>  [Data Source](https://gist.github.com/meperezcuello/82a9f1c1c473d6585e750ad2e3c05a41)  Import libraries and packages<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import altair as alt

import time

from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer

from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVR
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
from sklearn.feature_selection import RFECV


from sklearn.metrics import mean_absolute_error, mean_squared_error

a = {"x" : 1, "y" : 2}
pd.DataFrame.from_dict(a, "index")<jupyter_output><empty_output><jupyter_text>Load training data<jupyter_code>medical_data = pd.read_csv("../data/processed/medical_cost_data_training.csv")
X = medical_data.drop("charges", axis=1)
y = medical_data.charges
medical_data.head()
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)<jupyter_output><empty_output><jupyter_text>Identify categoric and numeric features<jupyter_code># Identify categoric and numeric features
def get_cat_num_features(x):
    data_types = x.dtypes
    categoric_features = []
    numeric_features = []
    for d_type, feature in zip(data_types, data_types.index):
        if d_type == "object":
            categoric_features.append(feature)
        else:
            numeric_features.append(feature)
    return categoric_features, numeric_features

categoric_features, numeric_features = get_cat_num_features(X)<jupyter_output><empty_output><jupyter_text>Create a preprocessing pipeline<jupyter_code>numeric_transformer = Pipeline(steps=[("poly" , PolynomialFeatures(degree=3)),
                                      ("scaler", StandardScaler())])
categoric_transformer = Pipeline(steps=[("ohe", OneHotEncoder())])

preprocessor = ColumnTransformer(transformers = [
                                 ("num", numeric_transformer, numeric_features),
                                 ("cat", categoric_transformer, categoric_features)])<jupyter_output><empty_output><jupyter_text>No Free Lunch - Try various models 1. linear regression2. Decisiontree regression3. knnregression4. Randomforestregression5. polynomial regression (Try polynomial features later)6. support vector machines<jupyter_code>models = {
    "linear_regression" : LinearRegression(),
    "decision_tree_regressor" : DecisionTreeRegressor(),
    "knn_regression" : KNeighborsRegressor(),
    "rf_regression" : RandomForestRegressor(n_estimators=100),
    "SVR" : SVR(gamma="scale")   
}
X_train.shape
y_train.shape
def try_models(models, X_train, y_train, preprocessor):
    results_dict = {}
    print(y_train.shape)
    for model_name, model in models.items():
#         print(f"trainning {model}")
        t = time.time()
        reg = Pipeline(steps=[('preprocessor', preprocessor),
                              ('regressor', model)])
        validation_error = np.mean(cross_val_score(reg, X_train, 
                                                   y_train, 
                                                   scoring = "neg_mean_absolute_error", 
                                                   cv=5) * -1)
        reg.fit(X_train, y_train)
        tr_err = mean_absolute_error(y_train, reg.predict(X_train))
        elapsed_time = time.time() - t
        results_dict[model_name] = [np.round(tr_err,3), np.round(validation_error,3), np.round(elapsed_time,4)]
        indices = ["Mean absolute error, training", "Mean absolute error, test", "training_time(s)"]
    results_df = pd.DataFrame(results_dict)
    results_df.index = indices
    return results_df
results_df = try_models(models, X_train, y_train, preprocessor)
results_df
pd.DataFrame(results_dict).T<jupyter_output><empty_output><jupyter_text>Hyperparameter optimization of DecisionTreeRegressor<jupyter_code>clf = Pipeline(steps=[('preprocessor', preprocessor),
                      ('regressor', DecisionTreeRegressor())])

param_grid = {
    'preprocessor__num__poly__degree': list(range(1, 5)),
    'regressor__max_depth': list(range(1, 20)),
    "regressor__min_samples_split" : list(range(2, 10))
}

grid_search = GridSearchCV(clf, param_grid, cv=5, return_train_score=True, scoring="neg_mean_absolute_error")
grid_search.fit(X, y)


print(mean_absolute_error(y_train, grid_search.predict(X_train)))
print(mean_absolute_error(y_test, grid_search.predict(X_test)))
grid_results.plot()
train_score = grid_search.cv_results_["mean_train_score"] * -1
test_score = grid_search.cv_results_["mean_test_score"] * -1
grid_results = pd.DataFrame({"train_score" : train_score, "test_score" : test_score})
# grid_results["max_depth"] = list(range(1, 20))
sns.lineplot("max_depth", "test_score", data=grid_results, color="r")
sns.lineplot("max_depth", "train_score", data=grid_results)<jupyter_output><empty_output><jupyter_text>Best model<jupyter_code>grid_search.best_params_
numeric_transformer = Pipeline(steps=[("poly" , PolynomialFeatures(degree=1)),
                                      ("scaler", StandardScaler())])
categoric_transformer = Pipeline(steps=[("ohe", OneHotEncoder())])

preprocessor = ColumnTransformer(transformers = [
                                 ("num", numeric_transformer, numeric_features),
                                 ("cat", categoric_transformer, categoric_features)])
reg = RandomForestRegressor(max_depth=5, min_samples_split=4)
model = Pipeline(steps = [("preprocessor", preprocessor),
                         ("regressor", reg)])

model.fit(X, y)
mean_absolute_error(y, model.predict(X))<jupyter_output><empty_output><jupyter_text>Load test data and test<jupyter_code>medical_data_test = pd.read_csv("../data/processed/medical_cost_data_test.csv")
X_test = medical_data_test.drop("charges", axis=1)
y_test = medical_data_test.charges
mean_absolute_error(y_test, model.predict(X_test))
grid = grid_search
y_predicted_training = grid.predict(X)
mean_absolute_error_trn = mean_absolute_error(y, y_predicted_training)
mean_squared_error_trn = mean_squared_error(y, y_predicted_training)
root_mean_squared_error_trn = np.sqrt(mean_squared_error(y, y_predicted_training))
y_predicted_test = grid.predict(X_test)
mean_absolute_error_test = mean_absolute_error(y_test, y_predicted_test)
mean_squared_error_test = mean_squared_error(y_test, y_predicted_test)
root_mean_squared_error_test = np.sqrt(mean_squared_error(y_test, y_predicted_test))
final_results = {"train" : [mean_absolute_error_trn, mean_squared_error_trn, root_mean_squared_error_trn],
"test" : [mean_absolute_error_test, mean_squared_error_test, root_mean_squared_error_test]}
df = pd.DataFrame(final_results)
df.index = ["Mean Absolute Error", "Mean Squared Error", "Root Mean Squared Error"]
df
def get_error(model, X, y):
    y_predicted = model.predict(X)
    mean_abs_error = mean_absolute_error(y, y_predicted)
    mean_sqrd_error = mean_squared_error(y, y_predicted)
    root_mean_sqrd_error = np.sqrt(mean_squared_error(y, y_predicted))
    return [mean_abs_error, mean_sqrd_error, root_mean_sqrd_error]

train_errors = get_error(grid, X, y)
test_errors = get_error(grid, X_test, y_test)

final_results = {"train" : train_errors,
                "test" : test_errors}
results_df = pd.DataFrame(final_results)
results_df.index = ["Mean Absolute Error", "Mean Squared Error", "Root Mean Squared Error"]
results_df
y_train_predicted = grid.predict(X)
y_test_predicted = grid.predict(X_test)
plt.scatter(y_test, y_test_predicted)
res = pd.DataFrame({"y_actual" : y_test, "y_predicted" : y_test_predicted, "residuals" : y_test - y_test_predicted})
res
alt.Chart(res).mark_circle(size=60, color="green", opacity=0.4).encode(
    alt.X('y_predicted', title = "Predicted Medical Expense ($)"),
    alt.Y('y_actual', title = "Actual Medical Expense ($)") 
    ).properties(
    title = "Predicted Vs Actual",
    width = 700,
    height = 300
    ).configure_axis(
    labelFontSize = 20,
    titleFontSize = 20
    ).configure_title(
    fontSize = 20
    )
p1 = alt.Chart(res).mark_circle(size=60, color="red", opacity=0.4).encode(
    alt.X('y_predicted', title = "Predicted Medical Expense ($)"),
    alt.Y('residuals', title = "Residuals ($)") 
    ).properties(
    title = "Residual Plot",
    width = 700,
    height = 300
    ).configure_axis(
    labelFontSize = 20,
    titleFontSize = 20
    ).configure_title(
    fontSize = 20
    )
plt.imshow(p1)
type(y)
type(preprocessor)
results_df.to_csv()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11078717201166181, 'fraction_numerical': 0.011325409284592958, 'mean_word_length': 4.732005141388175, 'pattern_counts': {'":': 1, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '630506', 'license': 'MIT', 'n_tokens_mistral': 3076, 'n_tokens_neox': 2919, 'n_words': 522, 'path': 'notebooks/medical_cost_prediction.ipynb', 'repo_name': 'sirine-chahma/DSCI_522_group_401', 'size': '8946'}