<jupyter_start><jupyter_text>Get Data<jupyter_code>query = """
SELECT b.avg_stars, b.length_, b.grade,
       r.star_ratings, r.on_to_do_lists, r.ticks
FROM routes b
LEFT JOIN ratings r ON b.url_id = r.url_id
WHERE b.area_name IN ('buttermilks', 'druid_stones', 'happy_boulders', 'sad_boulders')
  AND length_ IS NOT NULL
;
"""
data_df = pd_sql.read_sql(query, connection_uri) # grab data as a dataframe
data_df = data_df.reset_index(drop=True)
data_df.head()<jupyter_output><empty_output><jupyter_text>Linear Regression<jupyter_code># 5-Fold Cross-Validation
kf = KFold(n_splits=5, shuffle=True, random_state = 72)
scaler = StandardScaler()
# Set aside 20% of data for final test
features_train_all_df, features_test_all_df, target_train_ser, target_test_ser = train_test_split(data_df.drop(columns = ['on_to_do_lists']), data_df['on_to_do_lists'], test_size=.2, random_state=42)

# Choose features, KFold requires ndarray inputs
features_list = ['star_ratings', 'ticks', 'avg_stars', 'length_', 'grade']

features_train_chosen_df = features_train_all_df[features_list]
features_train_chosen_array = np.array(features_train_chosen_df)
features_test_chosen_df = features_test_all_df[features_list]
features_test_chosen_array = np.array(features_test_chosen_df)
target_train_array = np.array(target_train_ser)
target_test_array = np.array(target_test_ser)
# Scikit Learn's LinearRegression

# Initialize Model
linreg =  LinearRegression()
r2_cv_list = [] # Collect the cross-validation results

for train_index, val_cv_index in kf.split(features_train_chosen_array, target_train_array):
    
    # Make split
    features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
    features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_cv_index], target_train_array[val_cv_index] 
    
    # Standard Scaler on Features
    features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)
    features_val_cv_scaled_array = scaler.transform(features_val_cv_array)
    
    # Fit model
    linreg.fit(features_train_cv_scaled_array, target_train_cv_array)
    
    # Collect results
    SSE = sum((target_val_cv_array - linreg.predict(features_val_cv_scaled_array))**2)
    SST = sum((target_val_cv_array - np.mean(target_val_cv_array))**2)
    r2_cv_score = 1-SSE/SST
    r2_cv_list.append(r2_cv_score)

print(r2_cv_list)
print('r2_cv_score: ', mean(r2_cv_list),' +/- ', stdev(r2_cv_list))

# Test
features_train_scaled_array = scaler.fit_transform(features_train_chosen_df)
features_test_scaled_array = scaler.transform(features_test_chosen_df)
linreg.fit(features_train_scaled_array, target_train_ser) # Fit model

SSE = sum((target_test_ser - linreg.predict(features_test_scaled_array))**2)
SST = sum((target_test_ser - np.mean(target_test_ser))**2)
r2_test_score = 1-SSE/SST

print('r2_test_score: ', r2_test_score) #Get score
print(linreg.coef_)
# Statsmodels OLS

r2_cv_list = [] # Collect the cross-validation results

for train_index, val_cv_index in kf.split(features_train_chosen_array, target_train_array):
    
    # Make the split
    features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
    features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_cv_index], target_train_array[val_cv_index] 
    
    # Rebalance train features, no need to rebalance validation set, resample is better than smote
    features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)  # Scale resampled features
    features_val_cv_scaled_array = scaler.transform(features_val_cv_array)  # Scale validation features
    
    ols_model = sm.OLS(target_train_cv_array, sm.add_constant(features_train_cv_scaled_array))
    ols_results = ols_model.fit() # Fit model
    
    # Collect results on the validation set
    SSE = sum((target_val_cv_array - ols_results.predict(sm.add_constant(features_val_cv_scaled_array)))**2)
    SST = sum((target_val_cv_array - np.mean(target_val_cv_array))**2)
    r2_cv_list.append(1-SSE/SST)

print(r2_cv_list)
print('val_r2_score: ', mean(r2_cv_list),' +/- ', stdev(r2_cv_list))

# Test
features_train_scaled_array = scaler.fit_transform(features_train_chosen_df)
features_test_scaled_array = scaler.transform(features_test_chosen_df)

# Fit model
ols_model = sm.OLS(target_train_ser, sm.add_constant(features_train_scaled_array))
ols_results = ols_model.fit()

# Collect results on the test set
SSE = sum((target_test_ser - ols_results.predict(sm.add_constant(features_test_scaled_array)))**2)
SST = sum((target_test_ser - np.mean(target_test_ser))**2)

print('test_r2_score: ', 1-SSE/SST) #Get score
print(list(ols_results.params[1:]))
print(ols_results.summary())
x = ols_results.predict(sm.add_constant(features_test_scaled_array))
y = target_test_ser
feature_target_df = pd.DataFrame({'predicted': x, 'observed': y})

fig = plot_single_scatter(feature_target_df, 'predicted', 'observed',
                          xlabel='Predicted', ylabel='Observed',
                          color=None)
plt.plot([0, 500], [0, 500], c='orange', lw = 2, ls='--', label= "Equal")

if save:
    plt.savefig("figures/linear/observed-vs-predicted.png", transparent=True, bbox_inches="tight")<jupyter_output><empty_output><jupyter_text>Ridge<jupyter_code># Ridge Gridsearch

alpha_list = 10**(np.linspace(-2,5,201)) # Generate a list of 10^(x) 
mae_list, r2_list, coefs_list = [], [], []

for i,curr_alpha in enumerate(alpha_list):
    
    # Initialize Model
    ridgereg = Ridge(alpha = curr_alpha, random_state=123)
    mae_cv_list, r2_cv_list, coefs_cv_list = [], [], [] # Collect the cross-validation results
    
    for train_index, val_index in kf.split(features_train_chosen_array, target_train_array):
        
        # Make split
        features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
        features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_index], target_train_array[val_index] 

        # Standard Scaler on Features
        features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)
        features_val_cv_scaled_array = scaler.transform(features_val_cv_array)

        # Fit model
        ridgereg.fit(features_train_cv_scaled_array, target_train_cv_array)
        target_val_cv_predict_array = ridgereg.predict(features_val_cv_scaled_array)
        
        # Collect results
        mae_cv_list.append(np.mean(np.abs(target_val_cv_predict_array - target_val_cv_array)))
        r2_cv_list.append(ridgereg.score(features_val_cv_scaled_array, target_val_cv_array))
        coefs_cv_list.append(ridgereg.coef_)
        
    # Collect Results Average for K-Fold Cross Validation
    mae_list.append(np.mean(mae_cv_list))
    r2_list.append(mean(r2_cv_list))
    coefs_cv_array = np.stack(coefs_cv_list, axis = 0)
    coefs_cv_mean_array = np.mean(coefs_cv_array, axis = 0)
    coefs_list.append(coefs_cv_mean_array)
    
# Print alpha value that gave the lowest mean absolute error
alpha_min_mae = alpha_list[np.argmin(mae_list)]
print(alpha_min_mae)
# plot validation error with changing alpha
alpha_mae_df = pd.DataFrame({'alpha': alpha_list, 'mean_absolute_error': mae_list})
fig, ax = plot_empty(xlabel='alpha', ylabel='mean absolute error', figsize=None)
ax.plot(alpha_list, mae_list)
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1])
if save:
    plt.savefig("figures/ridge/mae-vs-alpha.png", transparent=True, bbox_inches="tight")
    
# Plot the coefficients as the alphas change
fig, ax = plot_empty(xlabel='alpha', ylabel='weights',
                     title='Ridge coefficients as a function of the regularization',
                     figsize=None,
                    )
ax.plot(alpha_list, coefs_list[:])
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1])  # reverse axis
plt.legend(features_train_chosen_df.columns)
plt.axis('tight')
if save:
    plt.savefig("figures/ridge/weights-vs-alpha.png", transparent=True, bbox_inches="tight")<jupyter_output><empty_output><jupyter_text>Optimal Ridge Regression<jupyter_code># Ridge Regression
ridgereg = Ridge(alpha=alpha_min_mae)

# KFold Cross-Validation
cv_r2_list = [] # Collect the cross-validation results

for train_index, val_cv_index in kf.split(features_train_chosen_array, target_train_array):
    
    # Make the split
    features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
    features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_cv_index], target_train_array[val_cv_index] 
    
    # Rebalance train features, no need to rebalance validation set, resample is better than smote
    features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)  # Scale resampled features
    features_val_cv_scaled_array = scaler.transform(features_val_cv_array)  # Scale validation features
    
    ridgereg.fit(features_train_cv_scaled_array, target_train_cv_array) # Fit model
    cv_r2_list.append(ridgereg.score(features_val_cv_scaled_array, target_val_cv_array)) # Collect results

print(cv_r2_list)
print('val_r2_score: ', mean(cv_r2_list),' +/- ', stdev(cv_r2_list))

# Test
features_train_scaled_array = scaler.fit_transform(features_train_chosen_df)
features_test_scaled_array = scaler.transform(features_test_chosen_df)
ridgereg.fit(features_train_scaled_array, target_train_ser) # Fit model

print('test_r2_score: ', ridgereg.score(features_test_scaled_array, target_test_ser)) #Get score
print(ridgereg.coef_)
x = ridgereg.predict(features_test_scaled_array)
y = target_test_ser
feature_target_df = pd.DataFrame({'predicted': x, 'observed': y})

fig = plot_single_scatter(feature_target_df, 'predicted', 'observed',
                          xlabel='Predicted', ylabel='Observed',
                          color=None)
plt.plot([0, 500], [0, 500], c='orange', lw = 2, ls='--', label= "Equal")

if save:
    plt.savefig("figures/ridge/observed-vs-predicted.png", transparent=True, bbox_inches="tight")<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code># Lasso Gridsearch

alpha_list = 10**(np.linspace(-5,0,201)) # Generate a list of 10^(x) 
mae_list, r2_list, coefs_list = [], [], []

for i,curr_alpha in enumerate(alpha_list):
    
    # Initialize Model
    lasso_model = Lasso(alpha = curr_alpha, random_state=123)
    
    mae_cv_list, r2_cv_list, coefs_cv_list = [], [], [] # Collect the cross-validation results
    
    for train_index, val_cv_index in kf.split(features_train_chosen_array, target_train_array):
        
        # Make split
        features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
        features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_cv_index], target_train_array[val_cv_index] 

        # Standard Scaler on Features
        features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)
        features_val_cv_scaled_array = scaler.transform(features_val_cv_array)

        # Fit model
        lasso_model.fit(features_train_cv_scaled_array, target_train_cv_array)
        target_val_cv_predict_array = lasso_model.predict(features_val_cv_scaled_array)
        
        # Collect results
        mae_cv_list.append(np.mean(np.abs(target_val_cv_predict_array - target_val_cv_array)))
        r2_cv_list.append(lasso_model.score(features_val_cv_scaled_array, target_val_cv_array))
        coefs_cv_list.append(lasso_model.coef_)
        
    # Collect Results Average for K-Fold Cross Validation
    mae_list.append(np.mean(mae_cv_list))
    r2_list.append(mean(r2_cv_list))
    coefs_cv_array = np.stack(coefs_cv_list, axis = 0)
    coefs_cv_mean_array = np.mean(coefs_cv_array, axis = 0)
    coefs_list.append(coefs_cv_mean_array)

# Print alpha value that gave the lowest mean absolute error
alpha_min_mae = alpha_list[np.argmin(mae_list)]
print(alpha_min_mae)
# plot validation error with changing alpha
alpha_mae_df = pd.DataFrame({'alpha': alpha_list, 'mean_absolute_error': mae_list})
fig, ax = plot_empty(xlabel='alpha', ylabel='mean absolute error', figsize=None)
ax.plot(alpha_list, mae_list)
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1])
if save:
    plt.savefig("figures/lasso/mae-vs-alpha.png", transparent=True, bbox_inches="tight")
    
# Plot the coefficients as the alphas change
fig, ax = plot_empty(xlabel='alpha', ylabel='weights',
                     title='Lasso coefficients as a function of the regularization',
                     figsize=None
                    )
ax.plot(alpha_list, coefs_list[:])
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1])  # reverse axis
plt.legend(features_train_chosen_df.columns)
plt.axis('tight')

if save:
    plt.savefig("figures/lasso/weights-vs-alpha.png", transparent=True, bbox_inches="tight")<jupyter_output><empty_output><jupyter_text>Optimize Lasso Regression<jupyter_code># Lasso
lasso_model = Lasso(alpha = alpha_min_mae)

# KFold Cross-Validation

cv_r2_list = [] # Collect the cross-validation results

for train_index, val_cv_index in kf.split(features_train_chosen_array, target_train_array):
    
    # Make the split
    features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
    features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_cv_index], target_train_array[val_cv_index] 
    
    # Rebalance train features, no need to rebalance validation set, resample is better than smote
    features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)  # Scale resampled features
    features_val_cv_scaled_array = scaler.transform(features_val_cv_array)  # Scale validation features
    
    lasso_model.fit(features_train_cv_scaled_array, target_train_cv_array) # Fit model
    cv_r2_list.append(lasso_model.score(features_val_cv_scaled_array, target_val_cv_array)) # Collect results
    
print(cv_r2_list)
print('val_r2_score: ', mean(cv_r2_list),' +/- ', stdev(cv_r2_list))

# Test
features_train_scaled_array = scaler.fit_transform(features_train_chosen_df)
features_test_scaled_array = scaler.transform(features_test_chosen_df)
lasso_model.fit(features_train_scaled_array, target_train_ser) # Fit model

print('test_r2_score: ', lasso_model.score(features_test_scaled_array, target_test_ser)) #Get score
print(lasso_model.coef_) #Get score
x = lasso_model.predict(features_test_scaled_array)
y = target_test_ser
feature_target_df = pd.DataFrame({'predicted': x, 'observed': y})

fig = plot_single_scatter(feature_target_df, 'predicted', 'observed',
                          xlabel='Predicted', ylabel='Observed',
                          color=None)
plt.plot([0, 500], [0, 500], c='orange', lw = 2, ls='--', label= "Equal")

if save:
    plt.savefig("figures/lasso/observed-vs-predicted.png", transparent=True, bbox_inches="tight")
# Compute regularization path for Lasso using the LARS
alphas_array, active_indices_list, coefs_array = lars_path(features_train_scaled_array, target_train_ser.values, method='lasso')
x = np.sum(np.abs(coefs_array.T), axis=1)
x /= x.max() # normalize by max

# Plot
fig = plot_empty('|coef| / max|coef|', 'Coefficients', 'LASSO Path')
plt.plot(x, coefs_array.T)
ymin, ymax = plt.ylim()
plt.vlines(x, ymin, ymax, linestyle='dashed')
plt.axis('tight')
plt.legend(features_train_chosen_df.columns)

if save:
    plt.savefig("figures/lasso/lasso-lars-path.png", transparent=True, bbox_inches="tight")<jupyter_output><empty_output><jupyter_text>ElasticNet<jupyter_code># ElasticNet
elastic_net = ElasticNet(random_state=1)

# KFold Cross-Validation
cv_r2_list = [] # Collect the cross-validation results

for train_index, val_cv_index in kf.split(features_train_chosen_array, target_train_array):
    
    # Make the split
    features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
    features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_cv_index], target_train_array[val_cv_index] 
    
    # Rebalance train features, no need to rebalance validation set, resample is better than smote
    features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)  # Scale resampled features
    features_val_cv_scaled_array = scaler.transform(features_val_cv_array)  # Scale validation features
    
    elastic_net.fit(features_train_cv_scaled_array, target_train_cv_array) # Fit model
    cv_r2_list.append(elastic_net.score(features_val_cv_scaled_array, target_val_cv_array)) # Collect results
    
print(cv_r2_list)
print('val_r2_score: ', mean(cv_r2_list),' +/- ', stdev(cv_r2_list))

# Test
features_train_scaled_array = scaler.fit_transform(features_train_chosen_df)
features_test_scaled_array = scaler.transform(features_test_chosen_df)
elastic_net.fit(features_train_scaled_array, target_train_ser) # Fit model

print('test_r2_score: ', elastic_net.score(features_test_scaled_array, target_test_ser)) #Get score
print(elastic_net.coef_)
# Statsmodels Elastic Net

r2_cv_list = [] # Collect the cross-validation results

for train_index, val_cv_index in kf.split(features_train_chosen_array, target_train_array):
    
    # Make the split
    features_train_cv_array, target_train_cv_array = features_train_chosen_array[train_index], target_train_array[train_index]
    features_val_cv_array, target_val_cv_array = features_train_chosen_array[val_cv_index], target_train_array[val_cv_index] 
    
    # Rebalance train features, no need to rebalance validation set, resample is better than smote
    features_train_cv_scaled_array = scaler.fit_transform(features_train_cv_array)  # Scale resampled features
    features_val_cv_scaled_array = scaler.transform(features_val_cv_array)  # Scale validation features
    
    ols_model = sm.OLS(target_train_cv_array, sm.add_constant(features_train_cv_scaled_array))
    ols_results = ols_model.fit_regularized(alpha=1, L1_wt = 0.5) # Fit model
    
    # Collect results on the validation set
    SSE = sum((target_val_cv_array - ols_results.predict(sm.add_constant(features_val_cv_scaled_array)))**2)
    SST = sum((target_val_cv_array - np.mean(target_val_cv_array))**2)
    r2_cv_list.append(1-SSE/SST)

print(r2_cv_list)
print('val_r2_score: ', mean(r2_cv_list),' +/- ', stdev(r2_cv_list))

# Test
features_train_scaled_array = scaler.fit_transform(features_train_chosen_df)
features_test_scaled_array = scaler.transform(features_test_chosen_df)

# Fit model
ols_model = sm.OLS(target_train_ser, sm.add_constant(features_train_scaled_array))
ols_results = ols_model.fit_regularized(alpha=1, L1_wt = 0.5)

# Collect results on the test set
SSE = sum((target_test_ser - ols_results.predict(sm.add_constant(features_test_scaled_array)))**2)
SST = sum((target_test_ser - np.mean(target_test_ser))**2)

print('test_r2_score: ', 1-SSE/SST) #Get score
print(list(ols_results.params[1:]))<jupyter_output>[0.6871345612574634, 0.6917005850597873, 0.49062326526523714, 0.5882322121580693, 0.7895719265480204]
val_r2_score:  0.6494525100577155  +/-  0.1138122813867808
test_r2_score:  0.6455970848299133
[19.750482848048794, 13.755394595294872, 8.233415129570187, 2.1220489439078367, 1.2687052588142056]
{'fraction_non_alphanumeric': 0.09927248335999174, 'fraction_numerical': 0.01945204065837676, 'mean_word_length': 5.807867931155602, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '398332', 'license': 'MIT', 'n_tokens_mistral': 7260, 'n_tokens_neox': 6894, 'n_words': 1086, 'path': 'train-ridge-and-lasso.ipynb', 'repo_name': 'harrisonized/mountain-project-recommender', 'size': '19381'}