<jupyter_start><jupyter_text>Lecture 34: VGGNet<jupyter_code>%matplotlib inline
import tqdm
import copy
import time
import torch
import numpy as np
import torchvision
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import matplotlib.pyplot as plt
from torchvision import transforms,datasets, models

print(torch.__version__) # This code has been updated for PyTorch 1.0.0<jupyter_output><empty_output><jupyter_text>Load Data:<jupyter_code>apply_transform = transforms.Compose([transforms.Resize(224),transforms.ToTensor()])
BatchSize = 4

trainset = datasets.CIFAR10(root='./CIFAR10', train=True, download=True, transform=apply_transform)
trainLoader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,
                                          shuffle=True, num_workers=4) # Creating dataloader

testset = datasets.CIFAR10(root='./CIFAR10', train=False, download=True, transform=apply_transform)
testLoader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,
                                         shuffle=False, num_workers=4) # Creating dataloader
# Size of train and test datasets
print('No. of samples in train set: '+str(len(trainLoader.dataset)))
print('No. of samples in test set: '+str(len(testLoader.dataset)))<jupyter_output><empty_output><jupyter_text>Define network architecture<jupyter_code>net = models.vgg16()
print(net)
# Counting number of trainable parameters
totalParams = 0
for name,params in net.named_parameters():
    print(name,'-->',params.size())
    totalParams += np.sum(np.prod(params.size()))
print('Total number of parameters: '+str(totalParams))
# Copying initial weights for visualization
init_weightConv1 = copy.deepcopy(net.features[0].weight.data) # 1st conv layer
init_weightConv2 = copy.deepcopy(net.features[2].weight.data) # 2nd conv layer
# Check availability of GPU

use_gpu = torch.cuda.is_available()
# use_gpu = False # Uncomment in case of GPU memory error
if use_gpu:
    print('GPU is available!')
    device = "cuda"
else:
    print('GPU is not available!')
    device = "cpu"
    
net = net.to(device)<jupyter_output><empty_output><jupyter_text>Define loss function and optimizer<jupyter_code>criterion = nn.NLLLoss() # Negative Log-likelihood
optimizer = optim.Adam(net.parameters(), lr=1e-4) # Adam<jupyter_output><empty_output><jupyter_text>Train the network<jupyter_code>iterations = 5
trainLoss = []
testAcc = []
start = time.time()
for epoch in range(iterations):
    epochStart = time.time()
    runningLoss = 0    
    net.train() # For training
    for data in tqdm.tqdm_notebook(trainLoader):
        inputs,labels = data
        inputs, labels = inputs.to(device), labels.to(device) 
       
        # Initialize gradients to zero
        optimizer.zero_grad()
        # Feed-forward input data through the network        
        outputs = net(inputs)        
        # Compute loss/error
        loss = criterion(F.log_softmax(outputs,dim=1), labels)
        # Backpropagate loss and compute gradients
        loss.backward()
        # Update the network parameters
        optimizer.step()
        # Accumulate loss per batch
        runningLoss += loss.item()  
        
    avgTrainLoss = runningLoss/(50000.0/BatchSize)
    trainLoss.append(avgTrainLoss)
    
    # Evaluating performance on test set for each epoch
    net.eval() # For testing [Affects batch-norm and dropout layers (if any)]
    running_correct = 0
    with torch.no_grad():
        for data in tqdm.tqdm_notebook(testLoader):
            inputs,labels = data
            inputs = inputs.to(device)
            outputs = net(inputs)
            _, predicted = torch.max(outputs.data, 1)
            if use_gpu:                
                predicted = predicted.cpu()            
            running_correct += (predicted == labels).sum()
            
    avgTestAcc = float(running_correct)*100/10000.0
    testAcc.append(avgTestAcc)
        
    # Plotting training loss vs Epochs
    fig1 = plt.figure(1)        
    plt.plot(range(epoch+1),trainLoss,'r-',label='train')        
    if epoch==0:
        plt.legend(loc='upper left')
        plt.xlabel('Epochs')
        plt.ylabel('Training loss')   
    # Plotting testing accuracy vs Epochs
    fig2 = plt.figure(2)        
    plt.plot(range(epoch+1),testAcc,'g-',label='test')        
    if epoch==0:
        plt.legend(loc='upper left')
        plt.xlabel('Epochs')
        plt.ylabel('Testing accuracy')    
    epochEnd = time.time()-epochStart
    print('Iteration: {:.0f} /{:.0f}  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\
          .format(epoch + 1,iterations,avgTrainLoss,avgTestAcc,epochEnd//60,epochEnd%60))
end = time.time()-start
print('Training completed in {:.0f}m {:.0f}s'.format(end//60,end%60))

# Copying trained weights for visualization
trained_weightConv1 = copy.deepcopy(net.features[0].weight.data)
trained_weightConv2 = copy.deepcopy(net.features[2].weight.data)
if use_gpu:
    trained_weightConv1 = trained_weightConv1.cpu()
    trained_weightConv2 = trained_weightConv2.cpu()<jupyter_output><empty_output><jupyter_text>Visualization of weights<jupyter_code># functions to show an image
def imshow(img, strlabel):
    npimg = img.numpy()
    npimg = np.abs(npimg)
    fig_size = plt.rcParams["figure.figsize"]
    fig_size[0] = 10
    fig_size[1] = 10
    plt.rcParams["figure.figsize"] = fig_size
    plt.figure()
    plt.title(strlabel)
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
imshow(torchvision.utils.make_grid(init_weightConv1,nrow=8,normalize=True),'Initial weights: conv1')
imshow(torchvision.utils.make_grid(trained_weightConv1,nrow=8,normalize=True),'Trained weights: conv1')
imshow(torchvision.utils.make_grid(init_weightConv1-trained_weightConv1,nrow=8,normalize=True),'Difference of weights: conv1')
imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Initial weights: conv2')
imshow(torchvision.utils.make_grid(trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Trained weights: conv2')
imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1)-trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Difference of weights: conv2')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12022992176273352, 'fraction_numerical': 0.019160146894459523, 'mean_word_length': 3.8520526723470176, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '19902', 'license': 'MIT', 'n_tokens_mistral': 2096, 'n_tokens_neox': 2023, 'n_words': 404, 'path': 'lecture34.ipynb', 'repo_name': 'snehashis1997/dlvcnptel', 'size': '6263'}