<jupyter_start><jupyter_text><jupyter_code>import numpy as np
from sklearn.datasets import load_boston
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sn

plt.rcParams["figure.figsize"] = (10,10)<jupyter_output><empty_output><jupyter_text>Load the dataset Lets make a dataset where $y = x +  \frac{3}{10} sin(x) + \frac{1}{10} \Big( log(x) \Big)^2 + \epsilon$,  where $\epsilon$ is a random number<jupyter_code>N_data = 5000

data_x = np.linspace(1.0, 10.0, N_data)[:, np.newaxis]
data_y = data_x + 0.2*np.random.randn(N_data,1) + 0.3*np.sin(data_x) + 1e-1  * np.log( data_x ) **2
#data_y = data_x + 0.1*np.random.randn(N_data,1) 
data_x.shape, data_y.shape
plt.plot( data_x.flatten(), data_y.flatten()  )
plt.xlabel('X')
plt.ylabel('Y')<jupyter_output><empty_output><jupyter_text>Make a train-test split of the available dataset<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(data_x, data_y, test_size=0.2, random_state=42)
X_train.shape, y_train.shape<jupyter_output><empty_output><jupyter_text>Define the Loss function \\$L = \frac{1}{2n} \sum_{i=1}^n \Big( y_i^{target} - y_i^{pred} \Big)^2$<jupyter_code># --- declare the L2 distance function between target & prefiction --- #
# def compute_Loss(y_tar, y_pred):

#     return 0.5 * np.mean( pow((y_pred- y_tar),2) )<jupyter_output><empty_output><jupyter_text>For a liner regression $y = Wx + b$ \\So, $\frac{\partial L}{\partial W} = \frac{1}{n} \sum_{i=1}^n \Big( y_i - y_i^{pred} \Big).x_i$ \\$\frac{\partial L}{\partial b} = \frac{1}{n} \sum_{i=1}^n \Big( y_i - y_i^{pred} \Big)$ The weights are updated following the rule : \\$W = W - \eta \frac{\partial L}{\partial W} $ \\$b = b - \eta \frac{\partial L}{\partial b} $ \\where $\eta$ is the learning rate<jupyter_code>tolerance = 1e-7 # numerical value to check if the gradient vanishes
class LinearRegression(object):
    def __init__(self, X, y, lr=0.001):

        self.lr = lr
        self.n_samples = len(y)
        self.n_features = X.shape[1]
        self.X = X
        self.y = y
        self.W = np.ones( (self.n_features ,1) )
        self.b = np.zeros( (1,1) )

    def fit(self, n_iter=1500): 

        for i in range(n_iter):

            y_pred = ( self.X ).dot( self.W)  + self.b

            loss = self.compute_Loss( self.y, y_pred )

            grad_weight =np.dot((y_pred-self.y).T,X_train)/self.n_samples

            grad_bias = np.mean(y_pred- self.y)

            if( abs( np.sum(grad_weight) ) < tolerance and abs(grad_bias) < tolerance ) : break

            self.W = self.W - self.lr * grad_weight.T
            self.b = self.b - self.lr * grad_bias


    def predict(self, X) : 

        y_pred = ( X ).dot( self.W)  + self.b
        return y_pred

    def params(self) : 

        weight_dict = {}
        weight_dict['weight'] = self.W
        weight_dict['bias'] = self.b

        return weight_dict
    
    # -- the L2 loss as a member function -- #
    def compute_Loss(self, y_tar, y_pred):

        return 0.5 * np.mean( pow((y_pred- y_tar),2) )
#del learn
learn = LinearRegression(X_train, y_train, lr=1e-3)
learn.fit(n_iter=2000)
y_pred = learn.predict( X_test)
y_pred.shape, y_test.shape
plt.scatter( y_test.flatten(), y_pred.flatten()  )
plt.xlabel('Target')
plt.ylabel('Prediction')

plt.plot( np.linspace(1,12, 1000), np.linspace(1,12, 1000), color='r', linewidth=1.5 )
learn.params()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1586483390607102, 'fraction_numerical': 0.02491408934707904, 'mean_word_length': 3.3123456790123456, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '539377', 'license': 'MIT', 'n_tokens_mistral': 1433, 'n_tokens_neox': 1363, 'n_words': 299, 'path': 'Tutorial1/Linear_Regression_Example.ipynb', 'repo_name': 'yish-gil/MLCourse_2021', 'size': '3492'}