<jupyter_start><jupyter_text>This notebook implements a very simple sequential network with pytorch. The dataset used is very small, and only used for some preliminary trials. The dataset is composed of three T1 images (class 0) and three FA images (class 1). The aim of this simple network is to classify images into the two classes. The training is done using a GPU and, in this example, only the central 10 axial slices are used.<jupyter_code>%matplotlib inline
from matplotlib import pyplot as plt
import numpy as np
import torch
import torchvision.transforms as transforms

torch.set_printoptions(edgeitems=2)
torch.manual_seed(0)
#mount the drive folder
from google.colab import drive
drive.mount('/content/drive') 
%cd /content/drive/My\ Drive/work/my_tests
#load the dataset as an instance of the class customDataset
from customDataset import HCPanatDataset

img_dir = 'HCP-anat-data/images/'
dataset = HCPanatDataset(csv_file='HCP-anat-data/annotations.csv', root_dir=img_dir)
#transform in torch tensor with transforms.ToTensor()
dataset_t = HCPanatDataset(csv_file='HCP-anat-data/annotations.csv', root_dir=img_dir, transform=transforms.ToTensor())
#compute the mean and std of the data
t1, _ = dataset[0]
max_dim = len(t1.shape) #concatenating dimension
imgs = torch.stack([img_t for img_t, _ in dataset_t], dim=max_dim)
#mean = torch.mean(imgs)
#std = torch.std(imgs)
#select some axial slices
print(imgs.shape) #z,y,y,n_sub
n_axial = 10
z_min = np.int(imgs.shape[0]/2) - np.int(n_axial/2)
z_max = np.int(imgs.shape[0]/2) + np.int(n_axial/2)
z_min, z_max
#crop dataset
n_crop = 20
imgs = imgs[z_min:z_max, n_crop:-n_crop, n_crop:-n_crop, :]
print(imgs.shape) #z,y,y,n_sub
#visualize an example of T1 cropped
t1 = imgs[0]
im=plt.imshow(t1[:,:,0])
plt.colorbar(im)
mean = torch.mean(imgs)
std = torch.std(imgs)
mean, std
#normalize the data
transformed_dataset = HCPanatDataset(
    csv_file='HCP-anat-data/annotations.csv', 
    root_dir=img_dir, 
    transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(mean,std)]))
#split the dataset into training and test sets with torch.utils.data.random_split
from torch.utils.data import random_split
N = len(transformed_dataset)
perc_train = 0.8
train_set, test_set = random_split(transformed_dataset, [int(perc_train*N), N-int(perc_train*N)]) 
print("Total number of images: %i" %N)
print("Number of training images: %i" %(perc_train*N))
print("Number of test images: %i" %(N-int(perc_train*N)))
#assuming that we are on a CUDA machine, this should print a CUDA device:
device = (torch.device('cuda') if torch.cuda.is_available() 
          else torch.device('cpu'))
print(f"Training on device {device}.")
#define the sequence of models with nn.Sequential()
import torch.nn as nn
n_in = imgs.shape[0] * imgs.shape[1] * imgs.shape[2] #number of input features
n_out = 2 #number of output features, i.e. number of classes
seq_model = nn.Sequential(
            nn.Linear(n_in, 128), 
            nn.Tanh(),
            nn.Linear(128, n_out))
seq_model = seq_model.to(device=device)
print("The number of input feature is: %i" %n_in)
#define the optimizer (SGD) and the loss (NLL)
import torch.optim as optim
optimizer = optim.SGD(seq_model.parameters(), lr=1e-3)
loss = nn.NLLLoss()
#define hyperparameters
n_epochs = 20
batch_size = 1
#split the datasets into batches
from torch.utils.data import DataLoader
train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(dataset=test_set, batch_size=batch_size, shuffle=True)
#define the training loop
def training_loop_val(model, train_loader, test_loader, criterion, optimizer, n_epochs):
  "Training loop with training and validation loss"
  loss_vector = np.zeros(n_epochs)
  loss_val_vector = np.zeros(n_epochs)
  
  for epoch in range(n_epochs):
    for imgs, labels in train_loader:
        imgs = imgs[:,z_min:z_max, n_crop:-n_crop, n_crop:-n_crop]
        imgs = imgs.to(device=device)
        labels = labels.to(device=device)
        outputs = model(imgs.view(imgs.shape[0], -1))
        loss = criterion(outputs, labels)
                
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    for imgs_test, labels_test in test_loader: 
        imgs_test = imgs_test[:,z_min:z_max, n_crop:-n_crop, n_crop:-n_crop]
        imgs_test = imgs_test.to(device=device)
        labels_test = labels_test.to(device=device)
        outputs_test = model(imgs_test.view(imgs_test.shape[0], -1))
        loss_test = criterion(outputs_test, labels_test)

    loss_vector[epoch] = float(loss)
    loss_val_vector[epoch] = float(loss_test)    
    print("Epoch: %d, Training Loss: %f, Validation Loss: %f" %(epoch, float(loss), float(loss_test)))    

  return loss_vector, loss_val_vector
for imgs, labels in train_loader:
    print(imgs.shape)
    imgs = imgs[:,z_min:z_max, n_crop:-n_crop, n_crop:-n_crop]
    print(imgs.shape)
#training and showing also validation loss
import time
t0 = time.time()
loss_vector, loss_val_vector = training_loop_val(
    model = seq_model,
    train_loader = train_loader,
    test_loader = test_loader,
    criterion = loss,
    optimizer = optimizer,
    n_epochs = n_epochs)
print("Training time = %f seconds" %(time.time()-t0))
#plot training and validation loss
x_axis = np.arange(n_epochs)
plt.plot(x_axis, loss_vector, 'r--', label='loss train')
plt.plot(x_axis, loss_val_vector, 'g--', label='loss val')
plt.ylim(-30, 30)
plt.legend()
#check accuracy
def check_accuracy(loader, model):

  model.eval()
  correct = 0
  total = 0

  with torch.no_grad():
    for imgs, labels in loader:
      imgs = imgs[:,z_min:z_max, n_crop:-n_crop, n_crop:-n_crop]
      imgs = imgs.to(device=device)
      labels = labels.to(device=device)
      outputs = model(imgs.view(imgs.shape[0], -1))
      _, predicted = torch.max(outputs, dim=1)
      total += labels.shape[0]
      correct += int((predicted == labels).sum())
        
  print("Accuracy: %f" % (correct / total))
#compute accuracy in training
check_accuracy(train_loader, seq_model)
#compute accuracy in test
check_accuracy(test_loader, seq_model)
#free some RAM
del seq_model
del dataset, dataset_t, transformed_dataset
del train_set, test_set, imgs
#increase the number of layers and change the loss to CrossEntropy
seq_model_large = nn.Sequential(
            nn.Linear(n_in, 128),
            nn.Tanh(),
            nn.Linear(128, 64),
            nn.Tanh(),
            nn.Linear(64, n_out))
seq_model_large = seq_model_large.to(device=device)
optimizer = optim.SGD(seq_model_large.parameters(), lr=1e-3)
loss_fn = nn.CrossEntropyLoss()
#training and showing also validation loss
import time
t0 = time.time()
loss_vector, loss_val_vector = training_loop_val(
    model = seq_model_large,
    train_loader = train_loader,
    test_loader = test_loader,
    criterion = loss_fn,
    optimizer = optimizer,
    n_epochs = n_epochs)
print("Training time = %f seconds" %(time.time()-t0))
#plot training and validation loss
x_axis = np.arange(n_epochs)
plt.plot(x_axis, loss_vector, 'r--', label='loss train')
plt.plot(x_axis, loss_val_vector, 'g--', label='loss val')
plt.ylim(0.03, 0.07)
plt.legend()
#compute accuracy in training
check_accuracy(train_loader, seq_model_large)
#compute accuracy in test
check_accuracy(test_loader, seq_model_large)
numel_list = [p.numel()
              for p in seq_model_large.parameters()
              if p.requires_grad == True]
sum(numel_list), numel_list<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11629452418926103, 'fraction_numerical': 0.01036682615629984, 'mean_word_length': 4.382689556509299, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '226086', 'license': 'MIT', 'n_tokens_mistral': 2720, 'n_tokens_neox': 2648, 'n_words': 672, 'path': 'notebooks/4_simple_sequence_nn_gpu_axial.ipynb', 'repo_name': 'giulia-berto/dl_with_pytorch', 'size': '7524'}