<jupyter_start><jupyter_text>XGBoost Regressor<jupyter_code>from xgboost import XGBRegressor

# Instansiate OrdinalEncoder
encoder = ce.OrdinalEncoder()

# encode X_train and X_val
X_train_encoded = encoder.fit_transform(X_train)
X_val_encoded = encoder.transform(X_val)
X_test_encoded = encoder.transform(X_test)

# Impute by hand to keep the dataframe as a dataframe
X_train_encoded['Engine'] = X_train_encoded['Engine'].fillna(X_train_encoded['Engine'].median())
X_train_encoded['Power_bhp'] = X_train_encoded['Power_bhp'].fillna(X_train_encoded['Power_bhp'].median())
X_train_encoded['Seats'] = X_train_encoded['Seats'].fillna(X_train_encoded['Seats'].median())

X_val_encoded['Engine'] = X_val_encoded['Engine'].fillna(X_val_encoded['Engine'].median())
X_val_encoded['Power_bhp'] = X_val_encoded['Power_bhp'].fillna(X_val_encoded['Power_bhp'].median())
X_val_encoded['Seats'] = X_val_encoded['Seats'].fillna(X_val_encoded['Seats'].median())

X_test_encoded['Engine'] = X_test_encoded['Engine'].fillna(X_test_encoded['Engine'].median())
X_test_encoded['Power_bhp'] = X_test_encoded['Power_bhp'].fillna(X_test_encoded['Power_bhp'].median())
X_test_encoded['Seats'] = X_test_encoded['Seats'].fillna(X_test_encoded['Seats'].median())

# insansiate XGBRegressor model
model = XGBRegressor(
    n_estimators = 1000,
    max_depth = 7,
    learning_rate = 0.5,
    n_jobs=-1
)

eval_set = [(X_train_encoded, y_train),
           (X_val_encoded, y_val)]

model.fit(X_train_encoded, y_train,
         eval_set=eval_set,
         eval_metric='mae',
         early_stopping_rounds=150)
y_pred = model.predict(X_val_encoded)

val_score = r2_score(y_val, y_pred)

print('Validation Score', val_score)
y_pred = model.predict(X_test_encoded)

val_score = r2_score(y_test, y_pred)

print('Validation Score', val_score)
# Using the XGB model to find the permutation importance in r^2
from eli5.sklearn import PermutationImportance

permuter = PermutationImportance(
    model,
    scoring='r2',
    n_iter=5,
    random_state=42
)

permuter.fit(X_val_encoded, y_val)
# Permutation Importance in r^2
feature_names = X_val.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values(ascending=False)
import eli5
eli5.show_weights(
    permuter,
    top=None,
    feature_names=feature_names
)
import shap
row = X_val_encoded.iloc[[0]]
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(row)

shap.initjs()
shap.force_plot(
    base_value=explainer.expected_value,
    shap_values=shap_values,
    features=row
)
from pdpbox.pdp import pdp_interact, pdp_interact_plot
features = ['Power_bhp', 'Year']

interaction = pdp_interact(
    model=model,
    dataset=X_val_encoded,
    model_features=X_val.columns,
    features=features
)
pdp_interact_plot(interaction, plot_type='grid', 
                  feature_names=features);<jupyter_output>findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.<jupyter_text>Random Forest Regressor<jupyter_code># Using RandomForestRegressor to fit my model without any feature engineering
# Accuracy is pretty high right off the bat, need to check for leakage
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestRegressor
from sklearn.pipeline import make_pipeline

pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='mean'),
    RandomForestRegressor(random_state=42, n_jobs=-1, n_estimators=1000)
)

pipeline.fit(X_train, y_train)
print('Validation Accuracy:', pipeline.score(X_val, y_val))
print('Test Accuracy:', pipeline.score(X_test, y_test))
from eli5.sklearn import PermutationImportance

permuter = PermutationImportance(
    pipeline,
    scoring='r2',
    n_iter=5,
    random_state=42
)

permuter.fit(X_val_encoded, y_val)
# feature_names = X_val.columns.tolist()
# pd.Series(permuter.feature_importances_, feature_names).sort_values(ascending=False)
import eli5
eli5.show_weights(
    permuter,
    top=None,
    feature_names=feature_names
)


import shap
row = X_val_encoded.iloc[[0]]
explainer = shap.TreeExplainer(pipeline)
shap_values = explainer.shap_values(row)

shap.initjs()
shap.force_plot(
    base_value=explainer.expected_value,
    shap_values=shap_values,
    features=row
)<jupyter_output><empty_output><jupyter_text>Linear Regression<jupyter_code>from sklearn.linear_model import LinearRegression
model = LinearRegression()
model.fit(X_train_encoded, y_train)
y_pred = model.predict(X_val_encoded)
r2_score(y_val, y_pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1113440972950304, 'fraction_numerical': 0.008177815055567205, 'mean_word_length': 6.511811023622047, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '252294', 'license': 'MIT', 'n_tokens_mistral': 1760, 'n_tokens_neox': 1706, 'n_words': 301, 'path': 'Build Week Unit 2.ipynb', 'repo_name': 'aklefebvere/DS-Unit-2-Applied-Modeling', 'size': '4769'}