<jupyter_start><jupyter_text>Linear Regression example<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import warnings 
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Generate a dataset<jupyter_code>my_data = pd.read_csv('ex1data1.txt', header=None)
my_data.columns = ['x', 'y']

x = my_data['x'].values
y = my_data['y'].values
 my_data.head()
my_data.describe() 
my_data.plot(kind='scatter', x='x', y='y')
def poly_regressors_fn(x_matrix, p_shape=1):
    # create list of ones with shape of x_matrix
    x_matrix_ones = np.ones((x_matrix.shape[0]))
    # create matrix with all values 1 as the first column
    x_poly = np.c_[x_matrix_ones, x_matrix]
    # power all columns by their respective values of i
    for i in range(2, p_shape + 1):        
        x_poly = np.c_[x_poly, x_matrix**i]
    return x_poly
def w_initial_fn(x_matrix, w_init=None):
    if w_init is None:
        # create array of zero values with shape of x_matrix
        return np.zeros(x_matrix.shape[1])
    else:
        return w_init
def gradient_descent(x, y, num_epochs, alpha, poly_order, w_initial=None):
    x_matrix = poly_regressors_fn(x, poly_order)
    w = w_initial_fn(x_matrix, w_initial)
    
    mse_history = []
    for epoch in range(num_epochs):
        # error is the values of y subtracted by the dot product between
        # x_matrix and w
        error = y - np.dot(x_matrix, w)
        # w is the sum of w multiplied by the mean of product between
        # error and x_matrix
        w += alpha * np.mean(error[:, None] * x_matrix, axis=0)
        # append list with history of mean square errors
        mse_history.append(np.mean((y - x_matrix @ w) ** 2))
    return {'w': w, 'mse_history': mse_history}
def st_gradient_descent(x, y, num_epochs, alpha, poly_order, w_initial=None):
    x_matrix = poly_regressors_fn(x, poly_order)
    w = w_initial_fn(x_matrix, w_initial)
    
    mse_history = []
    for epoch in range(num_epochs):
        parameters_p = np.random.permutation(y.shape[0])
        for x_p, y_p in zip(x[parameters_p], y[parameters_p]):
            # error is the values of y subtracted by the dot product between
            # x_matrix and w
            error = y_p - np.dot(w, x_p)
            # w is the sum of w multiplied by the mean of product between
            # error and x_matrix
            w += alpha * np.mean(error[:, None] * x_p, axis=0)
            # append list with history of mean square errors
            mse_history.append(np.mean((y_p - x_matrix @ w) ** 2))
    return {'w': w, 'mse_history': mse_history}
def ols_linear_regression(x, y, num_epochs, alpha, poly_order, w_initial=None):
    from numpy.linalg import inv
    
    x_matrix = poly_regressors_fn(x, poly_order)
    # create transpose of x matrix
    x_matrix_tr = x_matrix.transpose()
    w = w_initial_fn(x_matrix, w_initial)

    # compute ols equation, and return the parameters
    return np.dot(inv(np.dot(x_matrix_tr, x_matrix)), np.dot(x_matrix_tr, y))<jupyter_output><empty_output><jupyter_text>Gradient Descent<jupyter_code>alpha = 0.01
n_iter = 1000

print('[INFO] Running')

gd_results = gradient_descent(x, y, n_iter, alpha, my_data.shape[1] - 1)

w_gd1 = gd_results['w']    
mse_history_gd1 = gd_results['mse_history']
mse_gd1 = mse_history_gd1[-1]

print("Gradient Descent parameters 'w' = {}".format(w_gd1))
print("After {}, MSE: {}".format(n_iter, mse_gd1))
y_pred_plot1 = w_gd1[1] * x + w_gd1[0]
my_data.plot(kind='scatter', x='x',y='y')
plt.plot(x, y_pred_plot1, 'r')<jupyter_output>'c' argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with 'x' & 'y'.  Please use a 2-D array with a single row if you really want to specify the same RGB or RGBA value for all points.<jupyter_text>Stochastic Gradient Descent<jupyter_code>sgd_results = st_gradient_descent(x, y, n_iter, alpha, my_data.shape[1] - 1)
w_gd2 = sgd_results['w']    
mse_history_gd2 = sgd_results['mse_history']
mse_gd2 = mse_history_gd2[-1]

print("Stochastic Gradient Descent parameters 'w' = {}".format(w_gd2))
print("After {}, MSE: {}".format(n_iter, mse_gd2))
y_pred_plot2 = w_gd2[1] * x + w_gd2[0]
my_data.plot(kind='scatter', x='x',y='y')
plt.plot(x, y_pred_plot2, 'r')<jupyter_output>'c' argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with 'x' & 'y'.  Please use a 2-D array with a single row if you really want to specify the same RGB or RGBA value for all points.<jupyter_text>OLS Linear Regression<jupyter_code>ols_lr_results = ols_linear_regression(x, y, n_iter, alpha, my_data.shape[1] - 1)

w_lr = ols_lr_results
print("OLS Linear Regression parameters 'w' = {}".format(w_lr))
y_pred_plot3 = w_lr[1] * x + w_lr[0]
my_data.plot(kind='scatter', x='x',y='y')
plt.plot(x, y_pred_plot3, 'r')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11555377848860456, 'fraction_numerical': 0.011595361855257898, 'mean_word_length': 4.013026052104208, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '585545', 'license': 'MIT', 'n_tokens_mistral': 1863, 'n_tokens_neox': 1822, 'n_words': 495, 'path': '2_linear_regression_matrices.ipynb', 'repo_name': 'matheustguimaraes/machine-learning-discipline', 'size': '5002'}