<jupyter_start><jupyter_text>Ungraded Lab: Building a Custom Dense LayerIn this lab, we'll walk through how to create a custom layer that inherits the Layer class. Unlike simple Lambda layers you did previously, the custom layer here will contain weights that can be updated during training. Imports<jupyter_code>import math
import numpy as np
import torch
import torch.nn as nn
from torch.nn.parameter import Parameter<jupyter_output><empty_output><jupyter_text>Custom Layer with weightsTo make custom layer that is trainable, we need to define a class that inherits the nn.Module base class from PyTorch. The Python syntax is shown below in the class declaration. This class requires three functions: __init__() and forward(). These ensure that our custom layer has a state and computation that can be accessed during training or inference.<jupyter_code>class SimpleDense(nn.Module):
    def __init__(self, in_features, out_features, bias=True, device=None, dtype=None):
        super(SimpleDense, self).__init__()
        factory_kwargs = {'device': device, 'dtype': dtype}
        self.in_features = in_features
        self.bias = bias
        
        # Weight
        self.weight = Parameter(torch.empty((out_features, in_features), **factory_kwargs))
        
        # Bias
        if bias:
            self.bias = Parameter(torch.empty(out_features, **factory_kwargs))
        else:
            self.register_parameter('bias', None)
        
        # Weight and Biase initialization
        self._reset_parameters()
    
    def forward(self, input):
        x, y = input.shape
        if y != self.in_features:
            print(f'Wrong Input Features. Please use tensor with {self.in_features} Input Features')
            return 0
        output = input.matmul(self.weight.t())
        if self.bias is not None:
            output += self.bias
        ret = output
        return ret
    
    def _reset_parameters(self):
        torch.nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
        if self.bias is not None:
            fan_in, _ = torch.nn.init._calculate_fan_in_and_fan_out(self.weight)
            bound = 1 / math.sqrt(fan_in)
            torch.nn.init.uniform_(self.bias, -bound, bound)
# declare an instance of the class
my_dense = SimpleDense(in_features=1, out_features=1)

# define an input and feed into the layer
x = torch.ones((1, 1))
y = my_dense(x)

my_dense.state_dict()
# define the dataset
xs = np.array([-1.0,  0.0, 1.0, 2.0, 3.0, 4.0], dtype=np.float32).reshape(-1,1)
ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=np.float32).reshape(-1,1)

xs = torch.from_numpy(xs)
ys = torch.from_numpy(ys)

# Use the Sequential API to build a model with our custom layer
my_layer = SimpleDense(in_features=1, out_features=1)

modules = []
modules.append(my_layer)
model = nn.Sequential(*modules)

optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
criterion = torch.nn.MSELoss()


EPOCHS = 500

for epoch in range(EPOCHS):
    running_loss = 0
    
    optimizer.zero_grad()

    output = model(xs)
    loss = criterion(output, ys)
    loss.backward()

    optimizer.step()
    running_loss += loss.item()

print(f"Epoch: {epoch}, loss: {running_loss/len(xs)}")
model.eval()
x_test = torch.Tensor([[10.0]])
prediction = model(x_test)
prediction
# Updated Weight & Bias
my_layer.state_dict()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10619994067042421, 'fraction_numerical': 0.0154256897063186, 'mean_word_length': 3.2900763358778624, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '463118', 'license': 'MIT', 'n_tokens_mistral': 1095, 'n_tokens_neox': 1066, 'n_words': 310, 'path': 'Course-1 Custom Models, Layers and Loss Functions/Week-3/C1_W3_Lab_2_custom-dense-layer.ipynb', 'repo_name': 'anujdutt9/TensorFlow-Advanced-Techniques-Specilaization-in-PyTorch', 'size': '3371'}