<jupyter_start><jupyter_text>Application of ridge and lasso regression to find a separating hyperplane in a multivariate linear regression problem<jupyter_code>import numpy as np
import pandas as pd
import cvxpy as cp
df4 = pd.read_excel (r'regression.xlsx', header = None)
import matplotlib.pyplot as plt

# since b = 0, we do not need to worry about the intercept in the X matrix
X = df4.iloc[:,:8].values
y = df4.iloc[:,8].values
# define the loss function
def loss_fn(X, Y, a):
    return (cp.pnorm(Y - X @ a, p=2)**2)
# Linear regression

# Define and solve the CVXPY problem.
a = cp.Variable(df4.shape[1]-1)
cost = loss_fn(X, y, a)
prob = cp.Problem(cp.Minimize(cost))
prob.solve(solver=cp.SCS)
# Print result.
print("\nThe optimal value is", prob.value)
print("The optimal coefficients a are")
print(np.round(a.value, 5))
print("The norm of the residual is ", cp.norm(y - X*a, p=2).value)
results = pd.DataFrame(a.value)
results.to_csv(index=True, path_or_buf = 'results_linear.csv')
#print(results.to_latex(index=False)) 
# Before using lasso and ridge, we need to standardize the values in X and Y
y = (y - np.mean(y, axis=0)) / np.std(y, axis=0)
X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
# the regularizing function for lasso regression
def LASSO_regularize(a):
    return (cp.pnorm(a, p=1))

# a function to calculate the MSE
def mse(X, Y, a):
    return (1.0 / X.shape[0]) * loss_fn(X, Y, a).value
lambda_vals = np.array([1, 10, 100, 1000])
# Ridge regression
errors = []
a_values = []
for l in lambda_vals:
    # solve for astar using the unique solution formula
    astar = np.dot(np.linalg.inv((np.dot(X.T, X) + np.dot(l,np.identity(X.shape[1])))),(np.matmul(X.T ,y)))
    # Print result.
    a_values.append(astar)
    errors.append(mse(X, y, astar))
    print("The optimal coefficients a at lambda =", l, "are:")
    print(np.round(astar, 5))
results = pd.DataFrame(a_values).T
results.to_csv(index=True, path_or_buf = 'results_ridge.csv')
#print(results.to_latex(index=False)) 
s = plt.figure(figsize=[14,5])

def plot_errors(errors, lambda_vals):
    plot1 = plt.subplot(121) 
    plt.plot(lambda_vals, errors, label="Error")
    plt.xscale("log")
    plt.legend(loc="upper left")
    plt.xlabel(r"$\lambda$", fontsize=16)
    plt.title("Mean Squared Error (MSE)")
    
def plot_regularization_path(lambda_values, beta_values):
    plot2 = plt.subplot(122) 
    num_coeffs = len(a_values[0])
    for i in range(num_coeffs):
        plt.plot(lambda_vals, [wi[i] for wi in a_values])
    plt.xlabel(r"$\lambda$", fontsize=16)
    plt.xscale("log")
    plt.title("Regularization Path")

plot1 = plot_errors(errors, lambda_vals)
plot2 = plot_regularization_path(lambda_vals, a_values)

plt.show()

s.savefig('ridge.png')
# Lasso regression
errors = []
a_values = []
for l in lambda_vals:
    # Define and solve the CVXPY problem.
    a = cp.Variable(df4.shape[1]-1)
    cost = loss_fn(X, y, a) + l * LASSO_regularize(a)
    prob = cp.Problem(cp.Minimize(cost))
    prob.solve(solver=cp.SCS)
    errors.append(mse(X, y, a))
    a_values.append(a.value)
    # Print result.
    print("\nThe optimal value is", prob.value)
    print("The optimal coefficients a at lambda =", l, "are:")
    print(np.round(a.value, 5))
results = pd.DataFrame(a_values).T
results.to_csv(index=True, path_or_buf = 'results_lasso.csv')
#print(results.to_latex(index=False)) 
s = plt.figure(figsize=[14,5])

def plot_errors(errors, lambda_vals):
    plot1 = plt.subplot(121) 
    plt.plot(lambda_vals, errors, label="Error")
    plt.xscale("log")
    plt.legend(loc="upper left")
    plt.xlabel(r"$\lambda$", fontsize=16)
    plt.title("Mean Squared Error (MSE)")
    
def plot_regularization_path(lambda_values, beta_values):
    plot2 = plt.subplot(122) 
    num_coeffs = len(a_values[0])
    for i in range(num_coeffs):
        plt.plot(lambda_vals, [wi[i] for wi in a_values])
    plt.xlabel(r"$\lambda$", fontsize=16)
    plt.xscale("log")
    plt.title("Regularization Path")

plot1 = plot_errors(errors, lambda_vals)
plot2 = plot_regularization_path(lambda_vals, a_values)

plt.show()

s.savefig('lasso.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15041042974408497, 'fraction_numerical': 0.017624336069531628, 'mean_word_length': 4.311538461538461, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '251058', 'license': 'MIT', 'n_tokens_mistral': 1627, 'n_tokens_neox': 1566, 'n_words': 368, 'path': 'Ridge regression and LASSO regression for multivariate regression problem.ipynb', 'repo_name': 'singfie/Data-Driven-Optimization', 'size': '4142'}