<jupyter_start><jupyter_text>Test pipeline1. Create dataset: sequence of preporcessed examples ready to feed to neuralnet 2. Create dataloader: define how dataset is loaded to neuralnet (batch size, order, computation optimizing ...)3. Create model : a bunch of matrixes math to transform input tensor to output tensor4. Training loop:    + Forward     + Calculate loss    + Backward    + Monitoring:         + Evaluate metrics        + Logger, back and forth        + Visualize  Import necessary packages<jupyter_code>import os
import glob
import sys
import random

import matplotlib.pylab as plt
from PIL import Image, ImageDraw

import torch
import torch.nn.functional as F
import torch.nn as nn
from torch.utils.data import Dataset
import torchvision.transforms.functional as TF
from torchvision import datasets, models, transforms

import numpy as np

from sklearn.model_selection import ShuffleSplit

torch.manual_seed(0)
np.random.seed(0)
random.seed(0)

%matplotlib inline

sys.path.insert(0, '..')
from src.models.utils import FaceDataset<jupyter_output><empty_output><jupyter_text>Create a transformer<jupyter_code>data_transforms = {
    'train': transforms.Compose([
        transforms.Resize((224, 224),interpolation=Image.NEAREST),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize((224, 224),interpolation=Image.NEAREST),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

def transformer(image, label, params):
    if params["train"]:
        transform = data_transforms["train"]
    else:
        transform = data_transforms["val"]
    image = transform(image) 
    return image, label<jupyter_output><empty_output><jupyter_text>Create Data loader<jupyter_code>trans_params_train = {
    "train" : True,
}


trans_params_val={
    "train" : False,
}
path2data = "/home/Data/appa-real/processed/"


# create data set
train_ds = FaceDataset(path2data + "train.csv", transformer, trans_params_train)
val_ds = FaceDataset(path2data + "valid.csv", transformer, trans_params_val)
test_ds = FaceDataset(path2data + "test.csv", transformer, trans_params_val)
print(len(train_ds))
print(len(val_ds))
print(len(test_ds))
import matplotlib.pyplot as plt
def show(img,label=None):
    npimg = img.numpy().transpose((1,2,0))
    plt.imshow(npimg)
    if label is not None:
        label = label.view(-1,2)
        for point in label:
            x,y= point
            plt.plot(x,y,'b+',markersize=10)

plt.figure(figsize=(10,10))
for img,label in train_ds:
    show(img,label)
    break
plt.figure(figsize=(10,10))
for img,label in val_ds:
    show(img,label)
    break
from torch.utils.data import DataLoader
train_bz = 200
val_bz = 400
train_dl = DataLoader(train_ds, batch_size = train_bz, shuffle=True)
val_dl = DataLoader(val_ds, batch_size = val_bz, shuffle=False)
test_dl = DataLoader(test_ds, batch_size = val_bz, shuffle=False)
for img_b, label_b in train_dl:
    print(img_b.shape,img_b.dtype)
    print(label_b.shape)
    break
for img_b, label_b in val_dl:
    print(img_b.shape,img_b.dtype)
    print(label_b.shape)
    break
for img, label in test_dl:
    print(label.shape)
    break<jupyter_output>torch.Size([400, 2])<jupyter_text>Create Model<jupyter_code>class Net(nn.Module):
    def __init__(self, params_model):
        super(Net, self).__init__()
        
        # Loading Mobilenet v2 arch from PyTorch
        original_model = models.mobilenet_v2(pretrained=True)
        
        # Everything except the last linear layer
        self.features = nn.Sequential(*list(original_model.children())[:-1])
        
        # Get number of features of last layer
        num_feats = 1280
        
        # Plug our heads
        self.gender_head = nn.Sequential(
        nn.Linear(num_feats, 1)
        )
        
        self.age_head = nn.Sequential(
        nn.Linear(num_feats, 1)
        )
        
        if params_model["freeze"]:
        # Freeze all weights except the last classifier layer
            for p in self.features.parameters():
                p.requires_grad = False

    def forward(self, x):
        f = self.features(x)
        #f = F.adaptive_avg_pool2d(f,1)
        #f = f.view(f.size(0), -1)
        gender = F.sigmoid(self.gender_head(f))
        age = self.age_head(f)
        return age, gender
params_model={
        "freeze": False,
            }

model = Net(params_model)
device = torch.device("cuda")
model = nn.DataParallel(model)
model = model.to(device)
print(model)<jupyter_output>DataParallel(
  (module): Net(
    (features): Sequential(
      (0): Sequential(
        (0): ConvBNReLU(
          (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (2): ReLU6(inplace=True)
        )
        (1): InvertedResidual(
          (conv): Sequential(
            (0): ConvBNReLU(
              (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
              (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
              (2): ReLU6(inplace=True)
            )
            (1): Conv2d(32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (2): InvertedResidual(
          (conv): Sequential(
            (0): ConvBNReLU(
              (0): Conv[...]<jupyter_text>Create optimizer<jupyter_code>from torch import optim
from torch.optim.lr_scheduler import ReduceLROnPlateau


opt = optim.Adam(model.parameters(), lr=1e-4)
lr_scheduler = ReduceLROnPlateau(opt, mode='min',factor=0.5, patience=10,verbose=1)<jupyter_output><empty_output><jupyter_text>Create custom Performance<jupyter_code>from src.models import experiment
from src.models.experiment import Performance

class CustomPerformance(Performance):
    def __init__(self):
        Performance.__init__(self)
        self.gender_loss_func = nn.BCELoss(reduction="sum")
        self.age_loss_func = nn.MSELoss(reduction="sum")
        
    def loss_function(self, predictions, targets):
        age, gender = predictions
        gender_targets = targets[:, 1].unsqueeze(1)
        age_targets = targets[:, 0].unsqueeze(1)
        
        loss_age = self.age_loss_func(age, age_targets)
        
        loss_gender = self.gender_loss_func(gender, gender_targets)
        
        loss = loss_age + 100*loss_gender
        
        return loss

    def metrics_function(self, predictions, targets):
        age, gender = predictions
        gender_targets = targets[:, 1].unsqueeze(1)
        age_targets = targets[:, 0].unsqueeze(1)
        
        gender = gender > 0.5
        acc_gender = (gender == gender_targets).sum()
        
        mae_age = torch.abs(age - age_targets).sum()
        
        return torch.tensor([acc_gender, mae_age])<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>performance = CustomPerformance()
num_epochs = 100
path2models= "../models/weights.pt"
params = experiment.Prams(num_epochs=num_epochs, path2weights=path2models, device=device,optimizer=opt, lr_scheduler=lr_scheduler, sanity_check=False)

pipeline = experiment.Pipeline(model, train_dl, val_dl, performance, params)
model, performance = pipeline.train_val()

loss_hist, metrics_history = performance.loss_history, performance.metrics_history
train_hist = torch.stack(metrics_history["train"])
val_hist = torch.stack(metrics_history["val"])
# Train-Validation Progress

# plot loss progress
plt.title("Loss")
plt.plot(range(1,num_epochs+1),loss_hist["train"],label="train")
plt.plot(range(1,num_epochs+1),loss_hist["val"],label="val")
plt.ylabel("Loss")
plt.xlabel("Training Epochs")
plt.legend()
plt.show()
# plot accuracy progress
plt.title("Gender accuracy")
plt.plot(range(1,num_epochs+1),train_hist[:,0], label="train")
plt.plot(range(1,num_epochs+1),val_hist[:,0],label="val")
plt.ylabel("Acc")
plt.xlabel("Training Epochs")
plt.legend()
plt.show()
# plot accuracy progress
plt.title("Age MAE")
plt.plot(range(1,num_epochs+1),train_hist[:,1], label="train")
plt.plot(range(1,num_epochs+1),val_hist[:,1],label="val")
plt.ylabel("MAE")
plt.xlabel("Training Epochs")
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>loss_test, metrics_test = pipeline.test(test_dl)
# Gender Accuracy
print(max(train_hist[:,0]))
print(max(val_hist[:,0]))
print(metrics_test[0])
# Age MAE
print(min(train_hist[:,1]))
print(min(val_hist[:,1]))
print(metrics_test[1])
# Loss
print(min(loss_hist["train"]))
print(min(loss_hist["val"]))
print(loss_test)<jupyter_output>tensor(6.8821, device='cuda:0', grad_fn=<DivBackward0>)
tensor(93.0572, device='cuda:0')
tensor(109.0011, device='cuda:0')
{'fraction_non_alphanumeric': 0.1309510598157807, 'fraction_numerical': 0.028076795028298747, 'mean_word_length': 3.765732416710735, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '239902', 'license': 'MIT', 'n_tokens_mistral': 3245, 'n_tokens_neox': 3061, 'n_words': 605, 'path': 'notebooks/appa-real-age-gender-mobile-net-1.0.ipynb', 'repo_name': 'DucTranVan/face-attributes', 'size': '9011'}