<jupyter_start><jupyter_text>Test Uniform, Default and He Initialization on MNIST Dataset with Relu Activation Table of ContentsIn this lab, you will test the Uniform Initialization, Default Initialization and He Initialization on the MNIST dataset with Relu Activation    Neural Network Module and Training Function    Make Some Data    Define Several Neural Network, Criterion function, Optimizer    Test Uniform, Default and He Initialization    Analyze ResultsEstimated Time Needed: 25 min Preparation We'll need the following libraries:<jupyter_code># Import the libraries we need to use in this lab

# Using the following line code to install the torchvision library
# !conda install -y torchvision

import torch 
import torch.nn as nn
import torchvision.transforms as transforms
import torchvision.datasets as dsets
import torch.nn.functional as F
import matplotlib.pylab as plt
import numpy as np

torch.manual_seed(0)<jupyter_output><empty_output><jupyter_text>Neural Network Module and Training Function  Define the neural network module or class with He Initialization<jupyter_code># Define the class for neural network model with He Initialization

class Net_He(nn.Module):
    
    # Constructor
    def __init__(self, Layers):
        super(Net_He, self).__init__()
        self.hidden = nn.ModuleList()

        for input_size, output_size in zip(Layers, Layers[1:]):
            linear = nn.Linear(input_size, output_size)
            torch.nn.init.kaiming_uniform_(linear.weight, nonlinearity='relu')
            self.hidden.append(linear)

    # Prediction
    def forward(self, x):
        L = len(self.hidden)
        for (l, linear_transform) in zip(range(L), self.hidden):
            if l < L - 1:
                x = F.relu(linear_transform(x))
            else:
                x = linear_transform(x)
        return x<jupyter_output><empty_output><jupyter_text>Define the class or neural network with Uniform Initialization<jupyter_code># Define the class for neural network model with Uniform Initialization

class Net_Uniform(nn.Module):
    
    # Constructor
    def __init__(self, Layers):
        super(Net_Uniform, self).__init__()
        self.hidden = nn.ModuleList()

        for input_size, output_size in zip(Layers, Layers[1:]):
            linear = nn.Linear(input_size,output_size)
            linear.weight.data.uniform_(0, 1)
            self.hidden.append(linear)
    
    # Prediction
    def forward(self, x):
        L = len(self.hidden)
        for (l, linear_transform) in zip(range(L), self.hidden):
            if l < L - 1:
                x = F.relu(linear_transform(x))
            else:
                x = linear_transform(x)
                
        return x<jupyter_output><empty_output><jupyter_text>Class or Neural Network with PyTorch Default Initialization<jupyter_code># Define the class for neural network model with PyTorch Default Initialization

class Net(nn.Module):
    
    # Constructor
    def __init__(self, Layers):
        super(Net, self).__init__()
        self.hidden = nn.ModuleList()

        for input_size, output_size in zip(Layers, Layers[1:]):
            linear = nn.Linear(input_size, output_size)
            self.hidden.append(linear)
        
    def forward(self, x):
        L=len(self.hidden)
        for (l, linear_transform) in zip(range(L), self.hidden):
            if l < L - 1:
                x = F.relu(linear_transform(x))
            else:
                x = linear_transform(x)
                
        return x<jupyter_output><empty_output><jupyter_text>Define a function to train the model, in this case the function returns a Python dictionary to store the training loss and accuracy on the validation data<jupyter_code># Define function to  train model

def train(model, criterion, train_loader, validation_loader, optimizer, epochs = 100):
    i = 0
    loss_accuracy = {'training_loss': [], 'validation_accuracy': []}  
    
    #n_epochs
    for epoch in range(epochs):
        for i, (x, y) in enumerate(train_loader):
            optimizer.zero_grad()
            z = model(x.view(-1, 28 * 28))
            loss = criterion(z, y)
            loss.backward()
            optimizer.step()
            loss_accuracy['training_loss'].append(loss.data.item())
        
        correct = 0
        for x, y in validation_loader:
            yhat = model(x.view(-1, 28 * 28))
            _, label = torch.max(yhat, 1)
            correct += (label == y).sum().item()
        accuracy = 100 * (correct / len(validation_dataset))
        loss_accuracy['validation_accuracy'].append(accuracy)
    
    return loss_accuracy<jupyter_output><empty_output><jupyter_text>Make some Data  Load the training dataset by setting the parameters train  to True and convert it to a tensor  by placing a transform object int the argument transform<jupyter_code># Create the training dataset

train_dataset = dsets.MNIST(root='./data', train=True, download=True, transform=transforms.ToTensor())<jupyter_output><empty_output><jupyter_text>Load the testing dataset by setting the parameters train  False and convert it to a tensor  by placing a transform object int the argument transform<jupyter_code># Create the validation dataset

validation_dataset = dsets.MNIST(root='./data', train=False, download=True, transform=transforms.ToTensor())<jupyter_output><empty_output><jupyter_text>Create the training-data loader and the validation-data loader object<jupyter_code># Create the data loader for training and validation

train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=2000, shuffle=True)
validation_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=5000, shuffle=False)<jupyter_output><empty_output><jupyter_text>Define Neural Network, Criterion function, Optimizer and Train the Model  Create the criterion function<jupyter_code># Create the criterion function

criterion = nn.CrossEntropyLoss()<jupyter_output><empty_output><jupyter_text>Create a list that contains layer size<jupyter_code># Create the parameters

input_dim = 28 * 28
output_dim = 10
layers = [input_dim, 100, 200, 100, output_dim]<jupyter_output><empty_output><jupyter_text>Test PyTorch Default Initialization, Xavier Initialization and Uniform Initialization  Train the network using PyTorch Default Initialization<jupyter_code># Train the model with the default initialization

model = Net(layers)
learning_rate = 0.01
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
training_results = train(model, criterion, train_loader,validation_loader, optimizer, epochs=30)<jupyter_output><empty_output><jupyter_text>Train the network using He Initialization function<jupyter_code># Train the model with the He initialization

model_He = Net_He(layers)
optimizer = torch.optim.SGD(model_He.parameters(), lr=learning_rate)
training_results_He = train(model_He, criterion, train_loader, validation_loader, optimizer, epochs=30)<jupyter_output><empty_output><jupyter_text>Train the network using Uniform Initialization function<jupyter_code># Train the model with the Uniform initialization

model_Uniform = Net_Uniform(layers)
optimizer = torch.optim.SGD(model_Uniform.parameters(), lr=learning_rate)
training_results_Uniform = train(model_Uniform, criterion, train_loader, validation_loader, optimizer, epochs=30)<jupyter_output><empty_output><jupyter_text>Analyze Results  Compare the training loss for each activation<jupyter_code># Plot the loss

plt.plot(training_results_He['training_loss'], label='He')
plt.plot(training_results['training_loss'], label='Default')
plt.plot(training_results_Uniform['training_loss'], label='Uniform')
plt.ylabel('loss')
plt.xlabel('iteration ') 
plt.title('training loss iterations')
plt.legend()<jupyter_output><empty_output><jupyter_text>Compare the validation loss for each model<jupyter_code># Plot the accuracy

plt.plot(training_results_He['validation_accuracy'], label='He')
plt.plot(training_results['validation_accuracy'], label='Default')
plt.plot(training_results_Uniform['validation_accuracy'], label='Uniform') 
plt.ylabel('validation accuracy')
plt.xlabel('epochs ')   
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09011096207779538, 'fraction_numerical': 0.007560053651993659, 'mean_word_length': 4.164987405541562, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '70498', 'license': 'MIT', 'n_tokens_mistral': 2490, 'n_tokens_neox': 2355, 'n_words': 662, 'path': 'Coursera/IBM Python 01/Course04/8.3.3.He_Initialization_v2.ipynb', 'repo_name': 'brianshen1990/KeepLearning', 'size': '8201'}