<jupyter_start><jupyter_text>04_Model_Tuning  Import Libraries<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns 

from sklearn.preprocessing import  StandardScaler,PolynomialFeatures
from sklearn.impute import SimpleImputer

from sklearn.feature_selection import RFE
from sklearn.model_selection import train_test_split, cross_val_score, KFold, RandomizedSearchCV
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet


from sklearn.metrics import mean_squared_error<jupyter_output><empty_output><jupyter_text>In section 4, two strategies will be used.1. Transforming features by adding polynomial features from existing features    - Objective : Explore interactions between features2. Using Recursive Feature Elimination to reduce feature numbers    - Objective : Simplify Model  Strategy 1 : Transforming via Polynomial Features  Add polynomial features to existing set via `sklean.preprocessing.PolynomialFeatures`<jupyter_code>poly = PolynomialFeatures()
x_train_transformed_poly_names = poly.fit(x_train_transformed)
x_train_transformed_poly_names = x_train_transformed_poly_names.get_feature_names(x_train_transformed.columns)
x_train_transformed_poly = poly.transform(x_train_transformed)
x_test_transformed_poly = poly.transform(x_test_transformed)<jupyter_output><empty_output><jupyter_text>Reduce Model features using RFE<jupyter_code>rfe_ridge = RFE(ridge_result,n_features_to_select=6)
selector = rfe_ridge.fit(x_train_transformed_poly,y_train)
x_train_transformed_poly_names = np.array(x_train_transformed_poly_names)
poly_filter = x_train_transformed_poly_names[selector.get_support()]
new_columns = x_train_transformed_poly_names[selector.get_support()]
x_train_transformed_poly = pd.DataFrame(x_train_transformed_poly,columns=x_train_transformed_poly_names)
x_test_transformed_poly = pd.DataFrame(x_test_transformed_poly,columns=x_train_transformed_poly_names)
x_train_poly_transformed_rfe = x_train_transformed_poly[poly_filter]
x_test_poly_transformed_rfe = x_test_transformed_poly[poly_filter]
rfe_lm = LinearRegression()
rfe_lm = rfe_lm.fit(x_train_poly_transformed_rfe,y_train)
rfe_lm.score(x_test_poly_transformed_rfe,y_test)
y_predict_rfe =  rfe_lm.predict(x_test_poly_transformed_rfe)
draw_plot(y_predict_rfe,y_test)
metrics_printout(rfe_lm,x_test_transformed_rfe,y_test,y_predict_rfe)
rfe_ridge = Ridge()
rfe_alphas = np.logspace(0, 20, 1000)

rfe_params = {
    'alpha':rfe_alphas
}
random_search_ridge_rfe = RandomizedSearchCV(estimator=rfe_ridge, param_distributions=rfe_params,cv=5,n_iter=50)
ridge_search_rfe = random_search_ridge_rfe.fit(x_train_transformed_rfe,y_train)
ridge_search_rfe_alpha = ridge_search_rfe.best_params_['alpha']
rfe_ridge_optim = Ridge(alpha=ridge_search_rfe_alpha)
rfe_ridge_optim = rfe_ridge_optim.fit(x_train_transformed_rfe,y_train)
rfe_ridge_optim.score(x_test_transformed_rfe,y_test)
y_predict_rfe_ridge = rfe_ridge_optim.predict(x_test_transformed_rfe)
metrics_printout(rfe_ridge_optim,x_test_transformed_rfe,y_test,y_predict_rfe_ridge)<jupyter_output>R2 Correlation Score :0.8279461176344507
RMSE is : 32069.29246732327<jupyter_text>Strategy 2 : Reduce Model Features via RFE  Reduce Model features using RFE only<jupyter_code>rfe_ridge = RFE(ridge_result,n_features_to_select=7)
selector = rfe_ridge.fit(x_train_transformed,y_train)
x_train_transformed.columns[selector.get_support()]
new_columns = x_train_transformed.columns[selector.get_support()]
x_train_transformed_rfe = x_train_transformed[new_columns]
x_test_transformed_rfe = x_test_transformed[new_columns]
rfe_lm = LinearRegression()
rfe_lm = rfe_lm.fit(x_train_transformed_rfe,y_train)
rfe_lm.score(x_test_transformed_rfe,y_predict_rfe)
y_predict_rfe =  rfe_lm.predict(x_test_transformed_rfe)
plt.figure(figsize=(15,10))
ax = sns.scatterplot(x=y_test,y=y_predict_rfe,hue=y_test)
ax.set_title("Prediction of Ground Truth vs Predicted Price", fontname='Helvetica', fontsize=18,loc='left')
ax.set_xlabel('Ground Truth (Actual Price / $)',fontname='Helvetica',fontsize=12)
ax.set_ylabel('Predicted Price',fontname='Helvetica',fontsize=12)
plt.savefig("./img/business_predict.png",dpi=300)
metrics_printout(rfe_lm,x_test_transformed_rfe,y_test,y_predict_rfe)
rfe_lm.intercept_
rfe_lm.coef_
estimator_scaled_coefficients(rfe_lm,new_columns).sort_values(by='coefficient')
new_columns
rfe_ridge = Ridge()
rfe_alphas = np.logspace(0, 20, 1000)

rfe_params = {
    'alpha':rfe_alphas
}
random_search_ridge_rfe = RandomizedSearchCV(estimator=rfe_ridge, param_distributions=rfe_params,cv=5,n_iter=50)
ridge_search_rfe = random_search_ridge_rfe.fit(x_train_transformed_rfe,y_train)
ridge_search_rfe_alpha = ridge_search_rfe.best_params_['alpha']
rfe_ridge_optim = Ridge(alpha=ridge_search_rfe_alpha)
rfe_ridge_optim = rfe_ridge_optim.fit(x_train_transformed_rfe,y_train)
rfe_ridge_optim.score(x_test_transformed_rfe,y_test)
y_predict_rfe_ridge = rfe_ridge_optim.predict(x_test_transformed_rfe)
metrics_printout(rfe_ridge_optim,x_test_transformed_rfe,y_test,y_predict_rfe_ridge)<jupyter_output>R2 Correlation Score :0.8275674433549641
RMSE is : 32104.56380659157<jupyter_text>Export Ridge Coefficients for Production<jupyter_code>prod_model = pd.DataFrame(zip(new_columns,rfe_lm.coef_),columns=['feature','coefficient'])
prod_model = prod_model.append({'feature':'y_intercept','coefficient':rfe_lm.intercept_},ignore_index=True)
prod_model.to_csv('./datasets/cofficients_ridge.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0819552971106669, 'fraction_numerical': 0.01998909685626022, 'mean_word_length': 12.26265060240964, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '304951', 'license': 'MIT', 'n_tokens_mistral': 2134, 'n_tokens_neox': 2003, 'n_words': 253, 'path': 'project_2/04_Model_Tuning.ipynb', 'repo_name': 'bearylogical/ga-test', 'size': '5503'}