<jupyter_start><jupyter_text>Lab 01 : MNIST multi-layer -- demo<jupyter_code># For Google Colaboratory
import sys, os
if 'google.colab' in sys.modules:
    # mount google drive
    from google.colab import drive
    drive.mount('/content/gdrive')
    # find automatically the path of the folder containing "file_name" :
    file_name = 'mnist_multilayer_demo.ipynb'
    import subprocess
    path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode("utf-8")
    path_to_file = path_to_file.replace(file_name,"").replace('\n',"")
    # if previous search failed or too long, comment the previous line and simply write down manually the path below :
    #path_to_file = '/content/gdrive/My Drive/CS5242_2021_codes/codes/labs_lecture06/lab01_mnist_multilayer'
    print(path_to_file)
    # change current path to the folder containing "file_name"
    os.chdir(path_to_file)
    !pwd
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from random import randint
import time
import utils<jupyter_output><empty_output><jupyter_text>Download the data<jupyter_code>from utils import check_mnist_dataset_exists
data_path=check_mnist_dataset_exists()

train_data=torch.load(data_path+'mnist/train_data.pt')
train_label=torch.load(data_path+'mnist/train_label.pt')
test_data=torch.load(data_path+'mnist/test_data.pt')
test_label=torch.load(data_path+'mnist/test_label.pt')<jupyter_output><empty_output><jupyter_text>Make a two layer net class.<jupyter_code>class two_layer_net(nn.Module):

    def __init__(self, input_size, hidden_size,  output_size):
        super(two_layer_net , self).__init__()
        
        self.layer1 = nn.Linear(  input_size   , hidden_size  , bias=False  )
        self.layer2 = nn.Linear(  hidden_size  , output_size   , bias=False  )
        
        
    def forward(self, x):
        
        y       = self.layer1(x)
        y_hat   = F.relu(y)
        scores  = self.layer2(y_hat)
        
        return scores<jupyter_output><empty_output><jupyter_text>Build the net (recall that a one layer net had 7,840 parameters)<jupyter_code>net=two_layer_net(784,50,10)

print(net)
utils.display_num_param(net)<jupyter_output>two_layer_net(
  (layer1): Linear(in_features=784, out_features=50, bias=False)
  (layer2): Linear(in_features=50, out_features=10, bias=False)
)
There are 39700 (0.04 million) parameters in this neural network<jupyter_text>Choose the criterion, optimizer, batchsize, learning rate<jupyter_code>criterion = nn.CrossEntropyLoss()

optimizer=torch.optim.SGD( net.parameters() , lr=0.01 )

bs=20<jupyter_output><empty_output><jupyter_text>Evaluate on test set<jupyter_code>def eval_on_test_set():

    running_error=0
    num_batches=0

    for i in range(0,10000,bs):

        minibatch_data =  test_data[i:i+bs]
        minibatch_label= test_label[i:i+bs]

        inputs = minibatch_data.view(bs,784)

        scores=net( inputs ) 

        error = utils.get_error( scores , minibatch_label)

        running_error += error.item()

        num_batches+=1


    total_error = running_error/num_batches
    print( 'test error  = ', total_error*100 ,'percent')<jupyter_output><empty_output><jupyter_text>Training loop<jupyter_code>start = time.time()

for epoch in range(200):
    
    running_loss=0
    running_error=0
    num_batches=0
    
    shuffled_indices=torch.randperm(60000)
 
    for count in range(0,60000,bs):
        
        # forward and backward pass
    
        optimizer.zero_grad()
        
        indices=shuffled_indices[count:count+bs]
        minibatch_data =  train_data[indices]
        minibatch_label= train_label[indices]

        inputs = minibatch_data.view(bs,784)

        inputs.requires_grad_()

        scores=net( inputs ) 

        loss =  criterion( scores , minibatch_label) 
        
        loss.backward()

        optimizer.step()
        
        
        # compute some stats
        
        running_loss += loss.detach().item()
               
        error = utils.get_error( scores.detach() , minibatch_label)
        running_error += error.item()
        
        num_batches+=1
    
    
    # once the epoch is finished we divide the "running quantities"
    # by the number of batches
    
    total_loss = running_loss/num_batches
    total_error = running_error/num_batches
    elapsed_time = time.time() - start
    
    # every 10 epoch we display the stats 
    # and compute the error rate on the test set  
    
    if epoch % 5 == 0 : 
    
        print(' ')
        
        print('epoch=',epoch, '\t time=', elapsed_time,
              '\t loss=', total_loss , '\t error=', total_error*100 ,'percent')
        
        eval_on_test_set()<jupyter_output>epoch= 0 	 time= 1.5663700103759766 	 loss= 0.7022713695690036 	 error= 17.078333522876104 percent
test error  =  10.120000410079957 percent
 
epoch= 5 	 time= 9.432515144348145 	 loss= 0.22390149033349008 	 error= 6.216667409737905 percent
test error  =  6.070000624656678 percent
 
epoch= 10 	 time= 17.865931034088135 	 loss= 0.16048691512302807 	 error= 4.478334061304729 percent
test error  =  4.620000553131103 percent
 
epoch= 15 	 time= 26.04132318496704 	 loss= 0.12577241894525165 	 error= 3.4983339846134185 percent
test error  =  4.120000541210175 percent
 
epoch= 20 	 time= 34.57222294807434 	 loss= 0.10372214950512473 	 error= 2.8650005559126535 percent
test error  =  3.5500005245208746 percent
 
epoch= 25 	 time= 42.853976249694824 	 loss= 0.08861428492399864 	 error= 2.445000505447388 percent
test error  =  3.34000049829483 percent
 
epoch= 30 	 time= 51.1626660823822 	 loss= 0.07727210836469506 	 error= 2.1133337954680127 percent
test error  =  3.1100004792213443 percent
 [...]<jupyter_text>Choose image at random from the test set and see how good/bad are the predictions<jupyter_code># choose a picture at random
idx=randint(0, 10000-1)
im=test_data[idx]

# diplay the picture
utils.show(im)

# feed it to the net and display the confidence scores
scores =  net( im.view(1,784)) 
probs= F.softmax(scores, dim=1)
utils.show_prob_mnist(probs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09451765088661135, 'fraction_numerical': 0.09598177972994956, 'mean_word_length': 3.4518464880521362, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '32949', 'license': 'MIT', 'n_tokens_mistral': 2536, 'n_tokens_neox': 2137, 'n_words': 454, 'path': 'codes/labs_lecture06/lab01_mnist_multilayer/mnist_multilayer_demo.ipynb', 'repo_name': 'tanfiona/CS5242_2021', 'size': '6147'}