<jupyter_start><jupyter_text>Data Generating Process Consider a simple case as follows: $y = a+ b x + e$,where $y\in \mathbb{R}$, and $x \in \mathbb{R}$. Consider we observe $\{x_i,y_i\}_{i=1}^N$, and we want run the following regression$y_i  = \hat{a}+\hat{b} x_i$. We do this in two different method of opimization, (1) Gradient Descent(2) Stochastic Gradient Descent<jupyter_code>class Data_gen():
    def __init__(self, 
                 a =1.0,
                b = 1.0,
                N = 32,
                sigma = 0.1):
        torch.manual_seed(0)
        self.a= a
        self.b = b
        self.N = N
        self.sigma = sigma
        self.e = self.sigma*torch.randn(N)
        self.x = torch.arange(0.0,self.N,1)/self.N
        self.y = self.a+ self.b*self.x+self.e
    def train_data(self):
        data = torch.stack((self.x,self.y),1)
        return data

x = Data_gen().train_data()[:,[0]]
y = Data_gen().train_data()[:,[1]] 

plt.scatter(x,y, zorder=1,label= r"Data")
plt.ylabel(r"$Y$",fontsize=18)
plt.xlabel(r"$X$",fontsize=18)
plt.legend(loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>Finding the OLS estimators:$\hat{b} = \frac{\hat{cov}(x,y)}{\hat{var}(x)}$$\hat{a} = \hat{\mathbb{E}}[y]- \hat{b}~\hat{\mathbb{E}}(x)$<jupyter_code>dx = x-x.mean()
dy = y-y.mean()
cov = torch.matmul(dx.t(),dy)
var_x = torch.matmul(dx.t(),dx)
b_hat_ols = cov/var_x
a_hat_ols = y.mean() - b_hat_ols* x.mean()
print('b_hat=',b_hat_ols.item())
print('a_hat=',a_hat_ols.item())<jupyter_output>b_hat= 1.107523798942566
a_hat= 0.9485774040222168<jupyter_text>Designing the neural Net<jupyter_code>class NN(nn.Module):
    # This accepts an input of the form torch.Size([M, N])
    # M = sample size
    # N = number of firms (features)
    # output =  torch.Size([M, 1])
    def __init__(self,
                hidden_bias = True):
        super().__init__()
        self.hidden_bias = hidden_bias
        
        self.fc1 = nn.Linear(1,1, bias = self.hidden_bias)
        #hardcoding the initial weights
        self.fc1.weight.data = torch.tensor([[-1.5]])
        self.fc1.bias.data = torch.tensor([[3.0]])
    
    def forward(self, x):
        p_output = self.fc1(x)
        return  p_output<jupyter_output><empty_output><jupyter_text>Designing the dataloader<jupyter_code>class Dataset_loader(Dataset):
    def __init__(self):
        self.train_data = Data_gen().train_data()
        self.x = self.train_data[:,[0]]
        self.y = self.train_data[:,1:]
        self.n_samples = self.train_data.shape[0]
    
    def __getitem__(self,index):
        return self.x[index], self.y[index]
    
    def __len__(self):
        return self.n_samples
      
dataset = Dataset_loader()
dataloader_GD = DataLoader(dataset = dataset, batch_size = Data_gen().N , shuffle = False)
dataloader_SGD = DataLoader(dataset = dataset, batch_size = 1 , shuffle = True)<jupyter_output><empty_output><jupyter_text>Training Loop GD<jupyter_code>model = NN()

learning_rate = 1e-1
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0)

num_epochs = 100
weight_GD = []
bias_GD = []

for epoch in range(num_epochs):
    for i, (x,y) in enumerate(dataloader_GD):
        weight_GD.append(model.state_dict()['fc1.weight'].item())
        bias_GD.append(model.state_dict()['fc1.bias'].item())
        res = y - model(x)
        loss = res.pow(2).mean()
        optimizer.zero_grad() #this is essentially  with torch.no_grad():
        
        loss.backward()
        optimizer.step()
                
    #if epoch % 100 == 0:
    #    print(epoch,',',"{:.2e}".format(loss.detach().numpy()))       
        
        
bias_GD = np.array(bias_GD)
weight_GD = np.array(weight_GD)<jupyter_output><empty_output><jupyter_text>Training Loop SGD<jupyter_code>model = NN()

learning_rate = 1e-1
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0)

num_epochs = 10
weight_SGD = []
bias_SGD = []
for epoch in range(num_epochs):
    for i, (x,y) in enumerate(dataloader_SGD):
        weight_SGD.append(model.state_dict()['fc1.weight'].item())
        bias_SGD.append(model.state_dict()['fc1.bias'].item())
        res = y - model(x)
        loss = res.pow(2).mean()
        optimizer.zero_grad() #this is essentially  with torch.no_grad():    
        loss.backward()
        optimizer.step()
bias_SGD = np.array(bias_SGD)
weight_SGD = np.array(weight_SGD)
bias_SGD;<jupyter_output><empty_output><jupyter_text>Plotting<jupyter_code>plt.plot(weight_SGD,bias_SGD,'r+',label= r"Stochastic gradient descent")
plt.scatter(weight_GD,bias_GD, zorder=1,label= r"Gradient descent")
plt.ylabel(r"$\hat{\beta}$")
plt.xlabel(r"$\hat{\alpha}$")
plt.axhline(b_hat_ols,linewidth=1, color='k' )
plt.axvline(a_hat_ols,linewidth=1, color='k')
plt.legend(loc='best')
plt.show()<jupyter_output>The input object of type 'Tensor' is an array-like implementing one of the corresponding protocols (`__array__`, `__array_interface__` or `__array_struct__`); but not a sequence (or 0-D). In the future, this object will be coerced as if it was first converted using `np.array(obj)`. To retain the old behaviour, you have to either modify the type 'Tensor', or assign to an empty array created with `np.empty(correct_shape, dtype=object)`.<jupyter_text>Plotting $\hat{a}$<jupyter_code>fontsize= 16
ticksize = 14
figsize = (6, 3.5)
params = {"text.usetex": True,
    'font.family':'serif',
    "figure.figsize":figsize, 
    'figure.dpi': 80,
    'figure.edgecolor': 'k',
    'font.size': fontsize, 
    'axes.labelsize': fontsize,
    'axes.titlesize': fontsize,
    'xtick.labelsize': ticksize,
    'ytick.labelsize': ticksize
}
plt.rcParams.update(params)   
plt.plot(bias_SGD,label= r"Stochastic gradient descent")
plt.axhline(a_hat_ols.item(),linewidth=1, color='r', label = r"$\hat{a}_{OLS}$" )
plt.xlabel(r"Optimization Step")
plt.ylabel(r"$\hat{a}$")
plt.legend(loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>Plotting $\hat{b}$<jupyter_code>plt.plot(weight_SGD,label= r"Stochastic gradient descent")
plt.axhline(b_hat_ols.item(),linewidth=1, color='r', label = r"$\hat{b}_{OLS}$")
plt.ylabel(r"$\hat{b}$")
plt.xlabel(r"Optimization Step")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>Creating the gif<jupyter_code>fontsize= 16
ticksize = 14
figsize = (6, 3.5)
params = {"text.usetex": True,
    'font.family':'serif',
    "figure.figsize":figsize, 
    'figure.dpi': 80,
    'figure.edgecolor': 'k',
    'font.size': fontsize, 
    'axes.labelsize': fontsize,
    'axes.titlesize': fontsize,
    'xtick.labelsize': ticksize,
    'ytick.labelsize': ticksize
}
plt.rcParams.update(params) 
import os
import numpy as np
import matplotlib.pyplot as plt
import imageio
filenames = []
for i in range(len(weight_SGD)):
    # plot the line chart
    plt.plot(weight_SGD[:i],label= r"Stochastic gradient descent" )
    plt.axhline(b_hat_ols.item(),linewidth=1, color='r', label = r"$\hat{b}_{OLS}$")
    plt.ylabel(r"$\hat{b}$")
    plt.xlabel(r"Optimization Step")
    plt.legend(loc='lower right')
    plt.ylim(-2.0,1.2)
    plt.xlim(0,320)
    plt.tight_layout()
    # create file name and append it to a list
    filename = f'{i}.png'
    filenames.append(filename)
    
    # save frame
    plt.savefig(filename)
    plt.close()# build gif
with imageio.get_writer('SGD_LOS_b.gif', mode='I') as writer:
    for filename in filenames:
        image = imageio.imread(filename)
        writer.append_data(image)
        
# Remove files
for filename in set(filenames):
    os.remove(filename)
filenames = []
for i in range(len(bias_SGD)):
    # plot the line chart
    plt.plot(bias_SGD[:i],label= r"Stochastic gradient descent" )
    plt.axhline(a_hat_ols.item(),linewidth=1, color='r', label = r"$\hat{a}_{OLS}$")
    plt.ylabel(r"$\hat{a}$")
    plt.xlabel(r"Optimization Step")
    plt.legend(loc='upper right')
    plt.ylim(0.5,3.0)
    plt.xlim(0,320)
    plt.tight_layout()
    # create file name and append it to a list
    filename = f'{i}.png'
    filenames.append(filename)
    
    # save frame
    plt.savefig(filename)
    plt.close()# build gif
with imageio.get_writer('SGD_LOS_a.gif', mode='I') as writer:
    for filename in filenames:
        image = imageio.imread(filename)
        writer.append_data(image)
        
# Remove files
for filename in set(filenames):
    os.remove(filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15918270373010215, 'fraction_numerical': 0.01627464956046567, 'mean_word_length': 3.7112479015109123, 'pattern_counts': {'":': 4, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '240253', 'license': 'MIT', 'n_tokens_mistral': 3286, 'n_tokens_neox': 3145, 'n_words': 588, 'path': 'codes/stochastic gradient descent/1. Gradient Descent and Stochastic Gradient Descent.ipynb', 'repo_name': 'Mekahou/Fun-Stuff', 'size': '8418'}