<jupyter_start><jupyter_text>Foundations Of AIML Session 11 Experiment 2.1: Fine-tuning pre-trained CNN We have seen using the pre-trained model as a black box for feature extraction. This gave us a decent accuracy. However, if we have sufficent data we can *tweak* the learned model to extract features specific to our new dataset. Note that, we have 5000 training images which is not sufficient to train a deep model from scratch. But, 5000 might be enough to *tweak* the pre-trained model to be specific to our dataset. We will see what happens when we tweak only a small specific part of the pre-trained model. We will also see how to tweak the entire model. How many layers to tweak depends on amount of available data. Finetuning to specific data, when done properly, is almost always beneficial.<jupyter_code># Importing pytorch packages
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
import torchvision
import torchvision.transforms as transforms
from torch.autograd import Variable
# Importing config.py file
import config as cf
from utils import *
from light_cnn import network_9layers
from data_loader import *
## Importing python packages
import os
import sys
import time
import datetime
import numpy as np
import math
import matplotlib.pyplot as plt
# cf.data_dir = 'data/'
# img_root = cf.data_dir +'IIC/'
# chkpt=cf.data_dir+'checkpoint/'
# print(img_root, chkpt)
# experiment = 'lightnet_finetune_fc1_fc2_IMFDB'
# tmp=cf.data_dir+experiment
# print(tmp)
cf.data_dir = 'data/'
#img_root = cf.data_dir+'IMFDB_final/'
#train_list_file = cf.data_dir+'IMFDB_train.txt'   #### 5000 images for training
#val_list_file = cf.data_dir+'IMFDB_test.txt'      #### 1095 images for validation

# img_root = "data/IMFDB_final/"
# train_list_file = "data/IMFDB_train.txt"        #### 5000 images for training
# val_list_file = "data/IMFDB_test.txt"           #### 1095 images for validation
img_root = "data/IIC/"
train_list_file = "data/IIC_train.txt"
val_list_file = "data/IIC_test.txt"
print(img_root,train_list_file)

train_image_list = [line.rstrip('\n') for line in open(train_list_file)]
val_image_list = [line.rstrip('\n') for line in open(val_list_file)]

print (len(train_image_list), len(val_image_list))

# trainloader = torch.utils.data.DataLoader(custom_data_loader(img_root = img_root, image_list = train_list_file, crop=False,
#                                                              resize = True, resize_shape=[128,128]), 
#                                            batch_size=32, num_workers=16, shuffle = True, pin_memory=True)

# testloader = torch.utils.data.DataLoader(custom_data_loader(img_root = img_root, image_list = val_list_file, crop=False, mirror=False, 
#                                                            resize = True, resize_shape=[128,128]), 
#                                            batch_size=10, num_workers=5, shuffle = False, pin_memory=True)
trainloader = torch.utils.data.DataLoader(custom_data_loader(img_root = img_root, image_list = train_list_file, crop=False,
                                                             resize = True, resize_shape=[128,128],classes = 'IIC'), 
                                           batch_size=10, num_workers=0, shuffle = True, pin_memory=False)

testloader = torch.utils.data.DataLoader(custom_data_loader(img_root = img_root, image_list = val_list_file, crop=False, mirror=False, 
                                                           resize = True, resize_shape=[128,128],classes = 'IIC'), 
                                           batch_size=5, num_workers=0, shuffle = False, pin_memory=False)

#classes = ['AamairKhan', 'Rimisen', 'Kajol', 'KareenaKapoor','RishiKapoor', 'AmrishPuri', 'AnilKapoor', 'AnupamKher', 'BomanIrani', 'HrithikRoshan', 'KajalAgarwal', 'KatrinaKaif', 'Madhavan', 'MadhuriDixit', 'Umashri', 'Trisha']
classes = ['baba_ramdev', 'biswa',  'dhinchak_pooja',  'khali',  'priya_prakash']
# Checking for GPU instance
use_cuda = torch.cuda.is_available()
#Intilizaing the accuracy value as zero
best_acc = 0
#num_classes = 16
num_classes = 5
print(use_cuda)<jupyter_output>False<jupyter_text>Net surgerythe original pre-trained model has the last layer (fc2) for 79077 classes but we want to have last layer for only 16 classes.We chop-off the fc2 with 79077 classes and *implant* a new classifier (the MLP model we used in the previous experiment) which predicts 16 classes. Note that we could also implant a single FC layer with 16 classes (instead of a 3 layer MLP).<jupyter_code>feature_net = network_9layers()   ### creates an object of this network architecture
print(feature_net)
from collections import OrderedDict
new_state_dict = OrderedDict()
#state_dict = torch.load(cf.data_dir+'light_cnn/LightCNN_9Layers_checkpoint.pth.tar')['state_dict']    ### load the learned lightCNN weights
### load the learned lightCNN weights and map them to CPU since we dont have a GPU. The model weghts are trained on GPU and saved, so we need to remap to CPu 
state_dict = torch.load('data/light_cnn/LightCNN_9Layers_checkpoint.pth.tar',map_location=lambda storage, loc: storage)['state_dict']

for k, v in state_dict.items():
    name = k[7:] 
    new_state_dict[name] = v

feature_net.load_state_dict(new_state_dict)

### starting the surgery
layers_to_remove = ['fc2']
for layers_ in layers_to_remove:        
    del(feature_net._modules[layers_])
    
#### old fc2 removed.

classifier = nn.Sequential(nn.Linear(256, 64), nn.BatchNorm1d(64), nn.ReLU(),
                           nn.Linear(64, 32), nn.BatchNorm1d(32), nn.ReLU(),
                           nn.Linear(32, num_classes))

### implanting a new fc2
feature_net.fc2 = classifier
if use_cuda:
    feature_net.cuda()
    

print(feature_net)
### Intiliazing the loss
criterion = nn.CrossEntropyLoss()
def train(epoch):
    print('\nEpoch: %d' % epoch)
    feature_net.train()
    train_loss = 0
    correct = 0
    total = 0
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        print(len(inputs))
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()
        optimizer.zero_grad()
        inputs, targets = Variable(inputs), Variable(targets)
        outputs = feature_net(inputs)      ### notice that the pre-trained network has an implant classifier which directly outputs the 16 class prediction scores

        #print(outputs)
        #print(targets)
        size_ = outputs.size()
        #print(size_)
        outputs_ = outputs.view(size_[0], num_classes)
        #print(outputs_)
        loss = criterion(outputs_, targets)
        loss.backward()
        optimizer.step()

        train_loss += loss.data[0]
        _, predicted = torch.max(outputs_.data, 1)
        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()
        
        if batch_idx%10 == 0 or batch_idx == len(trainloader)-1:
            progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
        
    train_loss_file.write('%d %.3f %.3f\n' %(epoch, train_loss/len(trainloader), 100.*correct/total))


def test(epoch):
    global best_acc
    feature_net.eval()
    test_loss = 0
    correct = 0
    total = 0
    for batch_idx, (inputs, targets) in enumerate(testloader):
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()
        inputs, targets = Variable(inputs, volatile=True), Variable(targets)
        outputs = feature_net(inputs)
        size_ = outputs.size()
        outputs_ = outputs.view(size_[0], num_classes)
        loss = criterion(outputs_, targets)

        test_loss += loss.data[0]
        _, predicted = torch.max(outputs_.data, 1)
        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()
        
        if batch_idx%5 == 0 or batch_idx == len(testloader)-1:
            progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))
        
    print ('val_loss: ',  test_loss/len(testloader), 'accuracy: ', 100.0*correct/total)
    val_loss_file.write('%d %.3f %.3f\n' %(epoch,  test_loss/len(testloader), 100.*correct/total))

    # Save checkpoint.
    acc = 100.*correct/total
    if acc > best_acc:
        print('Saving..')
        state = {
            'net': classifier,
            'acc': acc,
            'epoch': epoch,
        }
        if not os.path.isdir(cf.data_dir+'checkpoint'):
            os.mkdir(cf.data_dir+'checkpoint')
        torch.save(state, cf.data_dir+'checkpoint/checkpoint_ckpt.t7')
        best_acc = acc
    
    return test_loss/len(testloader)
experiment = 'lightnet_finetune_fc1_fc2_IMFDB/'
#train_loss_file = open(cf.data_dir+experiment+"/train_loss.txt", "w", 0)
#val_loss_file = open(cf.data_dir+experiment+"/val_loss.txt", "w", 0)
train_loss_file = open("data/lightnet_finetune_fc1_fc2_IMFDB/train_loss.txt", "w+")
val_loss_file = open("data/lightnet_finetune_fc1_fc2_IMFDB/val_loss.txt", "w+")
### tweak only selected parts : FC1 and FC2. FC2 is 3 layer MLP.

layers_to_finetune = [{'params': feature_net.fc1.parameters()},
                      {'params':feature_net.fc2.parameters()}]

optimizer = optim.Adam(layers_to_finetune, lr=0.001)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=2, verbose=True)   #### dynamic LR scheduler
for epoch in range(0, 30):
    train(epoch)
    test_loss = test(epoch)
    scheduler.step(test_loss)
    
train_loss_file.close()
val_loss_file.close()
training_curves(cf.data_dir+experiment)
### After training we load the model that performed the best on validation data (avoid picking overfitted model)
classifier = torch.load(cf.data_dir+'checkpoint/checkpoint_ckpt.t7')['net'].eval()
def eval():
    feature_net.eval()
    
    testloader = torch.utils.data.DataLoader(custom_data_loader(img_root = img_root, image_list = val_list_file, crop=False, mirror=False, 
                                                           resize = True, resize_shape=[128,128],classes = 'IIC'), 
                                           batch_size=1, num_workers=0, shuffle = False, pin_memory=False)
    correct = 0
    total = 0
    conf_mat = np.zeros((num_classes, num_classes))
    total_ = np.zeros((num_classes))
    wrong_predictions = []
    for batch_idx, (inputs, targets) in enumerate(testloader):
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()
        inputs, targets = Variable(inputs, volatile=True), Variable(targets)
        outputs = feature_net(inputs)
        size_ = outputs.size()
        outputs_ = outputs.view(size_[0], num_classes)
        _, predicted = torch.max(outputs_.data, 1)
        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()
        prediction = predicted.cpu().numpy()[0]
        targets = targets.data.cpu().numpy()[0]
        total_[targets] +=1
        conf_mat[predicted, targets] +=1
        
        if prediction != targets:
            wrong_predictions += [[inputs, prediction, targets]]
        
    for k in range(num_classes):
        conf_mat[:,k] /= total_[k]
    return conf_mat, 100.*correct/total, wrong_predictions
    
conf, acc, wrong_predictions = eval()
print ('Accuracy:', acc, '%')<jupyter_output>C:\Users\vgopired\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel_launcher.py:15: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.
  from ipykernel import kernelapp as app<jupyter_text>Whoa!! :o Fine-tuning improved the accuracy by more than 15%<jupyter_code>plt.imshow(conf, cmap='jet', vmin=0, vmax = 1)
plt.show()
for w in wrong_predictions[::10]:
    print (classes[w[2]], 'confused with', classes[w[1]])
    plt.imshow(w[0][0][0].data.cpu().numpy(), cmap='gray')
    plt.show()<jupyter_output>baba_ramdev confused with khali<jupyter_text>Now let's try fine-tuning all layers in the network<jupyter_code>feature_net = network_9layers()
from collections import OrderedDict
new_state_dict = OrderedDict()
state_dict = torch.load(cf.data_dir+'light_cnn/LightCNN_9Layers_checkpoint.pth.tar',map_location=lambda storage, loc: storage)['state_dict']
for k, v in state_dict.items():
    name = k[7:] 
    new_state_dict[name] = v

feature_net.load_state_dict(new_state_dict)

layers_to_remove = ['fc2']
for layers_ in layers_to_remove:        
    del(feature_net._modules[layers_])
    


classifier = nn.Sequential(nn.Linear(256, 64), nn.BatchNorm1d(64), nn.ReLU(),
                           nn.Linear(64, 32), nn.BatchNorm1d(32), nn.ReLU(),
                           nn.Linear(32, num_classes))

feature_net.fc2 = classifier
if use_cuda:
    feature_net.cuda()
layers_to_finetune = [{'params': feature_net.features.parameters()}, 
                      {'params': feature_net.fc1.parameters()},
                      {'params':feature_net.fc2.parameters()}]
experiment = 'lightnet_finetune_all_IMFDB/'
train_loss_file = open(cf.data_dir+experiment+"train_loss.txt", "w+")
val_loss_file = open(cf.data_dir+experiment+"val_loss.txt", "w+")
best_acc = 0
optimizer = optim.Adam(layers_to_finetune, lr=0.001)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=2, verbose=True)   #### dynamic LR scheduler
for epoch in range(0, 30):
    train(epoch)
    test_loss = test(epoch)
    scheduler.step(test_loss)
    
train_loss_file.close()
val_loss_file.close()
training_curves(cf.data_dir+experiment)
classifier = torch.load(cf.data_dir+'checkpoint/checkpoint_ckpt.t7')['net'].eval()
conf, acc, wrong_predictions = eval()
print ('Accuracy:', acc, '%')
plt.imshow(conf, cmap='jet', vmin=0, vmax = 1)
plt.show()
for w in wrong_predictions[::10]:
    print (classes[w[2]], 'confused with', classes[w[1]])
    plt.imshow(w[0][0][0].data.cpu().numpy(), cmap='gray')
    plt.show()<jupyter_output>biswa confused with priya_prakash
{'fraction_non_alphanumeric': 0.12084485080445106, 'fraction_numerical': 0.019278474732440286, 'mean_word_length': 3.3697739238154227, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '593541', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 4757, 'n_tokens_neox': 4529, 'n_words': 1076, 'path': 'Session11/Lab11-Experiment2_2.ipynb', 'repo_name': 'sagargopireddy/AIML', 'size': '14109'}