<jupyter_start><jupyter_text>Performance on test sets<jupyter_code>import os
import torch
import numpy as np
import matplotlib.pyplot as plt
import random
from torch import nn
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms
from torchvision.datasets import MNIST
from tqdm import tqdm, tnrange
#%% Define the network architecture
class Autoencoder(nn.Module):
    def __init__(self, encoded_space_dim, dropout=0.2):
        super().__init__()
        
        ### Encoder
        self.encoder_cnn = nn.Sequential(
            nn.Conv2d(1, 8, 3, stride=2, padding=1),
            nn.ReLU(True),
            nn.Dropout(dropout),
            nn.Conv2d(8, 16, 3, stride=2, padding=1),
            nn.ReLU(True),
            nn.Dropout(dropout),
            nn.Conv2d(16, 32, 3, stride=2, padding=0),
            nn.ReLU(True),
            nn.Dropout(dropout)
        )
        self.encoder_lin = nn.Sequential(
            nn.Linear(3 * 3 * 32, 64),
            nn.ReLU(True),
            nn.Dropout(dropout),
            nn.Linear(64, encoded_space_dim)
        )
        
        ### Decoder
        self.decoder_lin = nn.Sequential(
            nn.Linear(encoded_space_dim, 64),
            nn.ReLU(True),
            nn.Dropout(dropout),
            nn.Linear(64, 3 * 3 * 32),
            nn.ReLU(True),
            nn.Dropout(dropout)
        )
        self.decoder_conv = nn.Sequential(
            nn.ConvTranspose2d(32, 16, 3, stride=2, output_padding=0),
            nn.ReLU(True),
            nn.Dropout(dropout),
            nn.ConvTranspose2d(16, 8, 3, stride=2, padding=1, output_padding=1),
            nn.ReLU(True),
            nn.Dropout(dropout),
            nn.ConvTranspose2d(8, 1, 3, stride=2, padding=1, output_padding=1)
        )

    def forward(self, x):
        x = self.encode(x)
        x = self.decode(x)
        return x

    def encode(self, x):
        # Apply convolutions
        x = self.encoder_cnn(x)
        # Flatten
        x = x.view([x.size(0), -1])
        # Apply linear layers
        x = self.encoder_lin(x)
        return x
    
    def decode(self, x):
        # Apply linear layers
        x = self.decoder_lin(x)
        # Reshape
        x = x.view([-1, 32, 3, 3])
        # Apply transposed convolutions
        x = self.decoder_conv(x)
        x = torch.sigmoid(x)
        return x
### Testing function
def test_net_v0(net, dataloader, loss_fn):

    # select device
    if torch.cuda.is_available():
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")
        
    # Validation
    net.eval() # Evaluation mode (e.g. disable dropout)
    with torch.no_grad(): # No need to track the gradients
        batch_len = np.zeros(len(dataloader))
        batch_loss = np.zeros(len(dataloader))
        for i, sample_batch in enumerate(dataloader,0):
            # Extract data and move tensors to the selected device
            image_batch = sample_batch[0].to(device)
            # Forward pass
            out = net(image_batch)
            
            # Evaluate global loss
            loss = loss_fn(out, image_batch)
            
            # save MSE loss and length of a batch
            batch_len[i] = len(sample_batch[0])
            batch_loss[i] = loss.item()
    
    # total MSE loss
    val_loss = (batch_loss*batch_len).sum()/batch_len.sum()
    return val_loss<jupyter_output><empty_output><jupyter_text>Uncorrupted inputs and outputs for different encoded dimensions<jupyter_code>#%% Define paths

data_root_dir = '../datasets'

#%% Create dataset

test_transform = transforms.Compose([
    transforms.ToTensor(),
])

test_dataset  = MNIST(data_root_dir, train=False, download=False, transform=test_transform)
test_loader = DataLoader(test_dataset, batch_size=512, shuffle=False)
encoded_space_dims = [1,2,4,8,16]
best_model_dict = np.load('best_model_dict.npy', allow_pickle=True).item()
loss_fn = nn.MSELoss()
test_losses = []
for encoded_space_dim in encoded_space_dims:
    # init autoencoder
    net = Autoencoder(encoded_space_dim, best_model_dict['dropout'])
    # load trained weights
    net.load_state_dict(torch.load('params/net_params_'+str(encoded_space_dim)+'.pth', 
                                   map_location=torch.device('cpu')))
    # test autoencoder
    test_loss = test_net_v0(net, test_loader, loss_fn)
    print("Encoded space dim: %d - loss : %.4f "%(encoded_space_dim,test_loss))
    test_losses.append(test_loss)
np.save('Analysis_results/test_losses', test_losses)
plt.figure(figsize=(8,6))
plt.plot(encoded_space_dims, test_losses, 'o-')
plt.xlabel("Encoded space dimension", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: uncorrupted - Test: uncorrupted", fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>Using noisy inputs but clean output targetsIdea: create a new dataset with each element given by (x_noisy, x_original)<jupyter_code>class NoisyDataset(Dataset):
    def __init__(self, dataset1, dataset2):
        
        assert len(dataset1) == len(dataset2), "Datasets should have the same length"
        data = []
        target = []
        labels = []
        s = dataset1[0][0].shape
        for i in range(len(dataset1)):
            data.append(dataset1[i][0])
            target.append(dataset2[i][0])
            #labels.append(dataset1[i][1])
        self.data = torch.cat(data).view((-1,)+s)
        self.target = torch.cat(target).view((-1,)+s)
        #self.labels = torch.LongTensor(labels)
        
    def __len__(self):
        return len(self.data)
    
    def __getitem__(self, index):
        return (self.data[index], self.target[index])#, self.labels[index])<jupyter_output><empty_output><jupyter_text>As for training we need a new test function.<jupyter_code>### Testing function
def test_denoisingAE(net, dataloader, loss_fn):

    # select device
    if torch.cuda.is_available():
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")
        
    # Validation
    net.eval() # Evaluation mode (e.g. disable dropout)
    with torch.no_grad(): # No need to track the gradients
        batch_len = np.zeros(len(dataloader))
        batch_loss = np.zeros(len(dataloader))
        for i, sample_batch in enumerate(dataloader,0):
            # Extract data and move tensors to the selected device
            data = sample_batch[0].to(device)
            target = sample_batch[1].to(device)
            # Forward pass
            out = net(data)
            
            # Evaluate global loss
            loss = loss_fn(out, target)
            
            # save MSE loss and length of a batch
            batch_len[i] = len(sample_batch[0])
            batch_loss[i] = loss.item()
    
    # total MSE loss
    val_loss = (batch_loss*batch_len).sum()/batch_len.sum()
    return val_loss<jupyter_output><empty_output><jupyter_text>Systematic analysis on gaussian noise intensity<jupyter_code># define loss
loss_fn = nn.MSELoss()
# define values of sigma to be tested
sigmas = np.linspace(0.01,0.5,20)

gaussian_test_losses = []
for encoded_space_dim in encoded_space_dims:
    print("Testing encoded space dim: %d"%encoded_space_dim)
    gaussian_losses = []
        
    # init autoencoder
    net = Autoencoder(encoded_space_dim, best_model_dict['dropout'])
    # load trained weights
    net.load_state_dict(torch.load('params/net_params_'+str(encoded_space_dim)+'.pth', 
                                   map_location=torch.device('cpu')))
    
    for sigma in sigmas:
        print('\r'+"Testing gaussian noising with sigma %.3f"%sigma, end=' ')

        # define/update gaussian transform with new sigma
        gaussian_test_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Lambda(lambda x: (x+np.random.normal(scale=sigma,size=x.shape)).float()),
        ])

        # load dataset and apply noise to it 
        gaussian_test_dataset = MNIST(data_root_dir, train=False, download=False, transform=gaussian_test_transform)
        # merge it with the original dataset that will be used as target
        gaussian_test_dataset = NoisyDataset(gaussian_test_dataset, test_dataset)
        # create test set loader to stream the data in batches to the net
        test_loader = DataLoader(gaussian_test_dataset, batch_size=512, shuffle=False)
        # compute the test loss
        gaussian_test_loss = test_denoisingAE(net, test_loader, nn.MSELoss())
        print('\r'+"Testing gaussian noising with sigma %.3f - loss : %.4f"%(sigma, gaussian_test_loss))
        gaussian_losses.append(gaussian_test_loss)
    gaussian_test_losses.append(gaussian_losses)
    
gaussian_test_losses = np.array(gaussian_test_losses)
np.save("Analysis_results/gaussian_losses", gaussian_test_losses)
np.save("Analysis_results/sigmas", sigmas)
plt.figure(figsize=(8,6))
for i in range(len(gaussian_test_losses)):
    plt.plot(sigmas,gaussian_test_losses[i,:], label='encoded dim %d'%(2**i))
plt.legend(fontsize=13)
plt.xlabel("Noise standard deviation", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: uncorrupted - Test: gaussian", fontsize=16)
plt.show()
plt.figure(figsize=(8,6))
plt.plot(encoded_space_dims,gaussian_test_losses[:,-1], '-o')
plt.xlabel("Encoded space dimension", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: uncorrupted - Test: gaussian", fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>Systematic analysis on scale of occlusion<jupyter_code># define loss
loss_fn = nn.MSELoss()
# define values of sigma to be tested
scales = np.linspace(0.01,0.5,20)

occluded_test_losses = []
for encoded_space_dim in encoded_space_dims:
    print("Testing encoded space dim: %d"%encoded_space_dim)
    occluded_losses = []
        
    # init autoencoder
    net = Autoencoder(encoded_space_dim, best_model_dict['dropout'])
    # load trained weights
    net.load_state_dict(torch.load('params/net_params_'+str(encoded_space_dim)+'.pth', 
                                   map_location=torch.device('cpu')))
    
    for scale in scales:
        print('\r'+"Testing occlusion noise with scale %.3f"%scale, end=' ')

        # define/update gaussian transform with new sigma
        occluded_test_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.RandomErasing(p=1, scale=(scale, scale), ratio=(1,1), value=np.random.rand()),
        ])

        # load dataset and apply noise to it 
        occluded_test_dataset = MNIST(data_root_dir, train=False, download=False, transform=occluded_test_transform)
        # merge it with the original dataset that will be used as target
        occluded_test_dataset = NoisyDataset(occluded_test_dataset, test_dataset)
        # create test set loader to stream the data in batches to the net
        test_loader = DataLoader(occluded_test_dataset, batch_size=512, shuffle=False)
        # compute the test loss
        occluded_test_loss = test_denoisingAE(net, test_loader, nn.MSELoss())
        print('\r'+"Testing occlusion noise with scale %.3f - loss : %.4f"%(scale, occluded_test_loss))
        occluded_losses.append(occluded_test_loss)
    occluded_test_losses.append(occluded_losses)
    
occluded_test_losses = np.array(occluded_test_losses)
np.save("Analysis_results/occluded_losses", occluded_test_losses)
np.save("Analysis_results/scales", scales)
plt.figure(figsize=(8,6))
for i in range(len(occluded_test_losses)):
    plt.plot(scales,occluded_test_losses[i,:], label='encoded dim %d'%(2**i))
plt.legend(fontsize=13)
plt.xlabel("Occlusion scale", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: uncorrupted - Test: occluded", fontsize=16)
plt.show()
plt.figure(figsize=(8,6))
plt.plot(encoded_space_dims, occluded_test_losses[:,-1], '-o')
plt.xlabel("Encoded space dimension", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: uncorrupted - Test: occluded", fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>Optional part: denoising autoencodersI know repeat the analysis on the corrupted test sets using 2 different kind of models: those trained on reconstructing images corrupted with gaussian noise and those trained on doing the same for occluded images.  Train: gaussian - Test: gaussian<jupyter_code># define loss
loss_fn = nn.MSELoss()
# define values of sigma to be tested
sigmas = np.linspace(0.01,0.5,20)

GG_test_losses = []
for encoded_space_dim in encoded_space_dims:
    print("Testing encoded space dim: %d"%encoded_space_dim)
    gaussian_losses = []
        
    # init autoencoder
    net = Autoencoder(encoded_space_dim, best_model_dict['dropout'])
    # load trained weights
    net.load_state_dict(torch.load('params/gaussian_net_params_'+str(encoded_space_dim)+'.pth', 
                                   map_location=torch.device('cpu')))
    
    for sigma in sigmas:
        print('\r'+"Testing gaussian noising with sigma %.3f"%sigma, end=' ')

        # define/update gaussian transform with new sigma
        gaussian_test_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Lambda(lambda x: (x+np.random.normal(scale=sigma,size=x.shape)).float()),
        ])

        # load dataset and apply noise to it 
        gaussian_test_dataset = MNIST(data_root_dir, train=False, download=False, transform=gaussian_test_transform)
        # merge it with the original dataset that will be used as target
        gaussian_test_dataset = NoisyDataset(gaussian_test_dataset, test_dataset)
        # create test set loader to stream the data in batches to the net
        test_loader = DataLoader(gaussian_test_dataset, batch_size=512, shuffle=False)
        # compute the test loss
        gaussian_test_loss = test_denoisingAE(net, test_loader, nn.MSELoss())
        print('\r'+"Testing gaussian noising with sigma %.3f - loss : %.4f"%(sigma, gaussian_test_loss))
        gaussian_losses.append(gaussian_test_loss)
    GG_test_losses.append(gaussian_losses)
    
GG_test_losses = np.array(GG_test_losses)
np.save("Analysis_results/GG_losses", GG_test_losses)
np.save("Analysis_results/sigmas", sigmas)
plt.figure(figsize=(8,6))
for i in range(len(GG_test_losses)):
    plt.plot(sigmas,GG_test_losses[i,:], label='encoded dim %d'%(2**i))
plt.legend(fontsize=13)
plt.xlabel("Noise standard deviation", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: gaussian - Test: gaussian", fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>Train: gaussian - Test: occluded<jupyter_code># define loss
loss_fn = nn.MSELoss()
# define values of sigma to be tested
scales = np.linspace(0.01,0.5,20)

GO_test_losses = []
for encoded_space_dim in encoded_space_dims:
    print("Testing encoded space dim: %d"%encoded_space_dim)
    occluded_losses = []
        
    # init autoencoder
    net = Autoencoder(encoded_space_dim, best_model_dict['dropout'])
    # load trained weights
    net.load_state_dict(torch.load('params/gaussian_net_params_'+str(encoded_space_dim)+'.pth', 
                                   map_location=torch.device('cpu')))
    
    for scale in scales:
        print('\r'+"Testing occlusion noise with scale %.3f"%scale, end=' ')

        # define/update gaussian transform with new sigma
        occluded_test_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.RandomErasing(p=1, scale=(scale, scale), ratio=(1,1), value=np.random.rand()),
        ])

        # load dataset and apply noise to it 
        occluded_test_dataset = MNIST(data_root_dir, train=False, download=False, transform=occluded_test_transform)
        # merge it with the original dataset that will be used as target
        occluded_test_dataset = NoisyDataset(occluded_test_dataset, test_dataset)
        # create test set loader to stream the data in batches to the net
        test_loader = DataLoader(occluded_test_dataset, batch_size=512, shuffle=False)
        # compute the test loss
        occluded_test_loss = test_denoisingAE(net, test_loader, nn.MSELoss())
        print('\r'+"Testing occlusion noise with scale %.3f - loss : %.4f"%(scale, occluded_test_loss))
        occluded_losses.append(occluded_test_loss)
    GO_test_losses.append(occluded_losses)
    
GO_test_losses = np.array(GO_test_losses)
np.save("Analysis_results/GO_losses", GO_test_losses)
np.save("Analysis_results/scales", scales)
plt.figure(figsize=(8,6))
for i in range(len(GO_test_losses)):
    plt.plot(scales,GO_test_losses[i,:], label='encoded dim %d'%(2**i))
plt.legend(fontsize=13)
plt.xlabel("Occlusion scale", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: gaussian - Test: occluded", fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>Train: occulded - Test: gaussian<jupyter_code># define loss
loss_fn = nn.MSELoss()
# define values of sigma to be tested
sigmas = np.linspace(0.01,0.5,20)

OG_test_losses = []
for encoded_space_dim in encoded_space_dims:
    print("Testing encoded space dim: %d"%encoded_space_dim)
    gaussian_losses = []
        
    # init autoencoder
    net = Autoencoder(encoded_space_dim, best_model_dict['dropout'])
    # load trained weights
    net.load_state_dict(torch.load('params/occluded_net_params_'+str(encoded_space_dim)+'.pth', 
                                   map_location=torch.device('cpu')))
    
    for sigma in sigmas:
        print('\r'+"Testing gaussian noising with sigma %.3f"%sigma, end=' ')

        # define/update gaussian transform with new sigma
        gaussian_test_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Lambda(lambda x: (x+np.random.normal(scale=sigma,size=x.shape)).float()),
        ])

        # load dataset and apply noise to it 
        gaussian_test_dataset = MNIST(data_root_dir, train=False, download=False, transform=gaussian_test_transform)
        # merge it with the original dataset that will be used as target
        gaussian_test_dataset = NoisyDataset(gaussian_test_dataset, test_dataset)
        # create test set loader to stream the data in batches to the net
        test_loader = DataLoader(gaussian_test_dataset, batch_size=512, shuffle=False)
        # compute the test loss
        gaussian_test_loss = test_denoisingAE(net, test_loader, nn.MSELoss())
        print('\r'+"Testing gaussian noising with sigma %.3f - loss : %.4f"%(sigma, gaussian_test_loss))
        gaussian_losses.append(gaussian_test_loss)
    OG_test_losses.append(gaussian_losses)
    
OG_test_losses = np.array(OG_test_losses)
np.save("Analysis_results/OG_losses", OG_test_losses)
np.save("Analysis_results/sigmas", sigmas)
plt.figure(figsize=(8,6))
for i in range(len(OG_test_losses)):
    plt.plot(sigmas,OG_test_losses[i,:], label='encoded dim %d'%(2**i))
plt.legend(fontsize=13)
plt.xlabel("Noise standard deviation", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: occluded - Test: gaussian", fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>Train: occulded - Test: occluded<jupyter_code># define loss
loss_fn = nn.MSELoss()
# define values of sigma to be tested
scales = np.linspace(0.01,0.5,20)

OO_test_losses = []
for encoded_space_dim in encoded_space_dims:
    print("Testing encoded space dim: %d"%encoded_space_dim)
    occluded_losses = []
        
    # init autoencoder
    net = Autoencoder(encoded_space_dim, best_model_dict['dropout'])
    # load trained weights
    net.load_state_dict(torch.load('params/gaussian_net_params_'+str(encoded_space_dim)+'.pth', 
                                   map_location=torch.device('cpu')))
    
    for scale in scales:
        print('\r'+"Testing occlusion noise with scale %.3f"%scale, end=' ')

        # define/update gaussian transform with new sigma
        occluded_test_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.RandomErasing(p=1, scale=(scale, scale), ratio=(1,1), value=np.random.rand()),
        ])

        # load dataset and apply noise to it 
        occluded_test_dataset = MNIST(data_root_dir, train=False, download=False, transform=occluded_test_transform)
        # merge it with the original dataset that will be used as target
        occluded_test_dataset = NoisyDataset(occluded_test_dataset, test_dataset)
        # create test set loader to stream the data in batches to the net
        test_loader = DataLoader(occluded_test_dataset, batch_size=512, shuffle=False)
        # compute the test loss
        occluded_test_loss = test_denoisingAE(net, test_loader, nn.MSELoss())
        print('\r'+"Testing occlusion noise with scale %.3f - loss : %.4f"%(scale, occluded_test_loss))
        occluded_losses.append(occluded_test_loss)
    OO_test_losses.append(occluded_losses)
    
OO_test_losses = np.array(OO_test_losses)
np.save("Analysis_results/OO_losses", OO_test_losses)
np.save("Analysis_results/scales", scales)
plt.figure(figsize=(8,6))
for i in range(len(OO_test_losses)):
    plt.plot(scales,OO_test_losses[i,:], label='encoded dim %d'%(2**i))
plt.legend(fontsize=13)
plt.xlabel("Occlusion scale", fontsize=16)
plt.ylabel("Test MSE loss", fontsize=16)
plt.title("Train: occluded - Test: occluded", fontsize=16)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11362781954887217, 'fraction_numerical': 0.013345864661654135, 'mean_word_length': 3.4877688738928723, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '474081', 'license': 'MIT', 'n_tokens_mistral': 7130, 'n_tokens_neox': 6876, 'n_words': 1477, 'path': 'Nicola_Dainese_Ex4_NN/Nicola_Dainese_Ex4_NN_pt2_Analysis.ipynb', 'repo_name': 'nicoladainese96/NeuralNetworks_DeepLearning', 'size': '21280'}