<jupyter_start><jupyter_text>Load dataset<jupyter_code>from utils import create_data_lists
voc07_path = os.path.join('datasets/', 'VOC2007/')
voc12_path = os.path.join('datasets/', 'VOC2012/')
create_data_lists(voc07_path, voc12_path, output_folder='datasets/')
!ls datasets/
torch.manual_seed(42)
trainset = VOCDataset(data_folder='datasets/', json_files=('TRAIN_images.json', 'TRAIN_objects.json'), augment=True, img_size=300)
valset = VOCDataset(data_folder='datasets/', json_files=('VAL_images.json', 'VAL_objects.json'), img_size=300)

dataloaders = dict(
    train = DataLoader(trainset, batch_size=32, collate_fn=collate_fn, shuffle=True, num_workers=2),
    val = DataLoader(valset, batch_size=32, collate_fn=collate_fn, shuffle=False, num_workers=2),
)<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>checkpoint = torch.load('../input/voc0712/pretrained_ssdeff.pt')
ssd_eff = checkpoint['model'].to(device)

ssd_eff.requires_grad_(False)
ssd_eff.lat_fm10.requires_grad_(True)
ssd_eff.lat_fm19.requires_grad_(True)
ssd_eff.final_fm19.requires_grad_(True)
ssd_eff.final_fm38.requires_grad_(True)
ssd_eff.det_fm38.requires_grad_(True)
ssd_eff.det_fm19.requires_grad_(True)
ssd_eff.det_fm10.requires_grad_(True)
ssd_eff.det_fm5.requires_grad_(True)
ssd_eff.det_fm3.requires_grad_(True)
checkpoint_path = gdrive_dir+'models/checkpoint_ssd_eff.pt'
checkpoint = torch.load(checkpoint_path) # None

if checkpoint:
    ssd_eff = checkpoint['model']
    optimizer = checkpoint['optimizer']
    #exp_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.96, last_epoch=checkpoint['epoch']-1)
else:
    ssd_eff = SSDEff(n_classes=21).to(device)
    optimizer = torch.optim.Adam(ssd_eff.parameters(), lr=2e-3, weight_decay=5e-4)
    #exp_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.96)

loss_func = MultiBoxLoss(priors_cxcy=ssd_eff.get_prior_boxes(), threshold=0.5, alpha=1., neg_pos_ratio=3, focal_loss=False)

grad_clip = None
def train_epoch(model, trainset_loader, loss_func, optimizer, epoch_id):
    model.train()
    train_loss = 0
    for step, (imgs, boxes, labels) in enumerate(trainset_loader):
        # move input data to GPU
        imgs = imgs.to(device)
        boxes = [b.to(device) for b in boxes]
        labels = [l.to(device) for l in labels]
        
        # forward
        predicted_offsets, predicted_scores = model(imgs)
        loss = loss_func(predicted_offsets, predicted_scores, boxes, labels)
        
        # backward & optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if epoch_id == 1 and step % 50 == 0:
            print(f'Epoch 1 - step {step}: train_loss: {loss.item():.4f}')
        
        train_loss += loss.item()
        
    return round(train_loss/len(trainset_loader), 4)

def eval_epoch(model, valset_loader, loss_func):
    model.eval()
    val_loss = 0
    with torch.no_grad():
        for step, (imgs, boxes, labels) in enumerate(valset_loader):
            imgs = imgs.to(device)
            boxes = [b.to(device) for b in boxes]
            labels = [l.to(device) for l in labels]
            
            predicted_offsets, predicted_scores = model(imgs)
            loss = loss_func(predicted_offsets, predicted_scores, boxes, labels)
            val_loss += loss.item()

    return round(val_loss/len(valset_loader), 4)


def train_model(model, dataloaders, optimizer, loss_func, lr_scheduler=None, n_epoch=1, checkpoint=None):
    last_epoch = 0
    if checkpoint:
        last_epoch = checkpoint['epoch']

    train_loss_hist, val_loss_hist = [], []
    for epoch_id in range(last_epoch+1, last_epoch+n_epoch+1):
        start_time = time.time()
        train_loss = train_epoch(model, dataloaders['train'], loss_func, optimizer, epoch_id)
        training_time = round(time.time() - start_time)
        if lr_scheduler: lr_scheduler.step()
        
        save_checkpoint(epoch_id, model, optimizer, checkpoint_path)
        
        # eval val_loss every epoch
        start_time = time.time()
        val_loss = eval_epoch(model, dataloaders['val'], loss_func) # if epoch_id % 5 == 0 else 'N/A'
        val_time = round(time.time() - start_time)
        
        train_loss_hist.append(train_loss)
        val_loss_hist.append(val_loss)
        
        print(f'Epoch {epoch_id} - train/val_time: {training_time}s | {val_time}s - train_loss: {train_loss} - val_loss: {val_loss}')
        
    return train_loss_hist, val_loss_hist

def adjust_learning_rate(optimizer, lr):
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5)
adjust_learning_rate(optimizer, lr=1.5e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=3, checkpoint=checkpoint)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=8, checkpoint=checkpoint)
checkpoint = torch.load(checkpoint_path)
adjust_learning_rate(optimizer, lr=1.25e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=10, checkpoint=checkpoint)
adjust_learning_rate(optimizer, lr=1e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=10, checkpoint=checkpoint)
adjust_learning_rate(optimizer, lr=0.75e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)
adjust_learning_rate(optimizer, lr=0.6e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)
adjust_learning_rate(optimizer, lr=0.4e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=10, checkpoint=checkpoint)
# Unfreeze extra conv blocks
ssd_eff.extras.requires_grad_(True)
adjust_learning_rate(optimizer, lr=0.2e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=10, checkpoint=checkpoint)
# There are a total of 26 blocks in the backbone
# Unfreeze block 20-25
for i in range(20,26):
    ssd_eff.backbone._blocks[i].requires_grad_(True)

adjust_learning_rate(optimizer, lr=0.1e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)
# Unfreeze block 13-19
for i in range(13,20):
    ssd_eff.backbone._blocks[i].requires_grad_(True)

adjust_learning_rate(optimizer, lr=0.06e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)
# Unfreeze block 5-12
for i in range(5,13):
    ssd_eff.backbone._blocks[i].requires_grad_(True)

adjust_learning_rate(optimizer, lr=0.04e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=10, checkpoint=checkpoint)
# Unfreeze the whole net for final fine-tuning
ssd_eff.backbone.requires_grad_(True)

adjust_learning_rate(optimizer, lr=0.03e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=10, checkpoint=checkpoint)
adjust_learning_rate(optimizer, lr=0.02e-3)
train_hist, val_hist = train_model(ssd_eff, dataloaders, optimizer, loss_func, n_epoch=5, checkpoint=checkpoint)<jupyter_output>Epoch 101 - train/val_time: 461s | 53s - train_loss: 1.9213 - val_loss: 2.3855
Epoch 102 - train/val_time: 458s | 52s - train_loss: 1.9137 - val_loss: 2.3867
Epoch 103 - train/val_time: 458s | 53s - train_loss: 1.9055 - val_loss: 2.3882
Epoch 104 - train/val_time: 459s | 53s - train_loss: 1.9052 - val_loss: 2.3787
Epoch 105 - train/val_time: 460s | 52s - train_loss: 1.9112 - val_loss: 2.3713<jupyter_text>Evaluate<jupyter_code>def eval_APs_mAP(model, valset_loader):
    '''
    '''
    model.eval()
    detected_boxes = list()
    detected_labels = list()
    detected_scores = list()
    true_boxes = list()
    true_labels = list()
    
    with torch.no_grad():
        for step, (imgs, boxes, labels) in enumerate(valset_loader):
            # move input data to GPU
            imgs = imgs.to(device)
            boxes = [b.to(device) for b in boxes]
            labels = [l.to(device) for l in labels]

            # detect objects
            predicted_offsets, predicted_scores = model(imgs)
            batch_det_boxes, batch_det_labels, batch_det_scores = model.post_process_top_k(predicted_offsets, predicted_scores,
                                                                                      score_threshold=0.02, iou_threshold=0.45, top_k=200)
            detected_boxes.extend(batch_det_boxes)
            detected_labels.extend(batch_det_labels)
            detected_scores.extend(batch_det_scores)
            true_boxes.extend(boxes)
            true_labels.extend(labels)

        # calculate metrics value
        detected_boxes_np = [i.cpu().numpy() for i in detected_boxes]
        detected_labels_np = [i.cpu().numpy() for i in detected_labels]
        detected_scores_np = [i.cpu().numpy() for i in detected_scores]
        true_boxes_np = [i.cpu().numpy() for i in true_boxes]
        true_labels_np = [i.cpu().numpy() for i in true_labels]
        result = eval_detection_voc(detected_boxes_np, detected_labels_np, detected_scores_np, true_boxes_np, true_labels_np)
    
    return result['ap'], result['map']
APs, mAP = eval_APs_mAP(ssd_eff, dataloaders['val'])
print(np.around(APs, decimals=3))
print(np.around(mAP, decimals=4))<jupyter_output>[  nan 0.857 0.869 0.755 0.702 0.452 0.777 0.841 0.912 0.62  0.805 0.734
 0.87  0.874 0.867 0.853 0.599 0.756 0.794 0.941 0.777]
0.7828<jupyter_text>Explore precision recall, choosing threshold value<jupyter_code>import matplotlib.pyplot as plt
from eval_detection_voc import calc_detection_voc_prec_rec

def cal_prec_rec(det_boxes, det_labels, det_scores, true_boxes, true_labels):
    det_boxes_np = [i.cpu().numpy() for i in det_boxes]
    det_labels_np = [i.cpu().numpy() for i in det_labels]
    det_scores_np = [i.cpu().numpy() for i in det_scores]
    true_boxes_np = [i.cpu().numpy() for i in true_boxes]
    true_labels_np = [i.cpu().numpy() for i in true_labels]
    
    precision, recall = calc_detection_voc_prec_rec(det_boxes_np, det_labels_np, det_scores_np, true_boxes_np, true_labels_np)
    precision = np.array([precision[i][-1] for i in range(1,21)])
    recall = np.array([recall[i][-1] for i in range(1,21)])
    return precision, recall
all_predicted_offsets = list()
all_predicted_scores = list()
true_boxes = list()
true_labels = list()

ssd_eff.eval()
with torch.no_grad():
    for step, (imgs, boxes, labels) in enumerate(dataloaders['val']):
        imgs = imgs.to(device)
        boxes = [b.to(device) for b in boxes]
        labels = [l.to(device) for l in labels]
        
        predicted_offsets, predicted_scores = ssd_eff(imgs)
        all_predicted_offsets.append(predicted_offsets)
        all_predicted_scores.append(predicted_scores)
        true_boxes.extend(boxes)
        true_labels.extend(labels)
        if step == 80: break
        
    all_predicted_offsets = torch.cat(all_predicted_offsets, dim=0)
    all_predicted_scores = torch.cat(all_predicted_scores, dim=0)
precs, recs = [], []

with torch.no_grad():
    for score_threshold in np.arange(0.02, 0.70, 0.02):
        det_boxes, det_labels, det_scores = ssd_eff.post_process_top_k(all_predicted_offsets, all_predicted_scores,
                                                                       score_threshold, iou_threshold=0.45, top_k=200)
        prec, rec = cal_prec_rec(det_boxes, det_labels, det_scores, true_boxes, true_labels)
        precs.append(prec)
        recs.append(rec)
precs_mean = [i.mean() for i in precs]
recs_mean = [i.mean() for i in recs]
plt.style.use("ggplot")
plt.figure()
plt.plot(recs_mean, precs_mean)
plt.title("Precision/Recall curve")
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.show()
print('threshold_value: precision  &  recall')
for threshold_id, threshold_value in enumerate(np.arange(0.02, 0.70, 0.02)):
    print(f'{threshold_value:.2f}:  {precs[threshold_id].mean():.2f}   {recs[threshold_id].mean():.2f}')<jupyter_output>threshold_value: precision  &  recall
0.02:  0.09   0.96
0.04:  0.17   0.93
0.06:  0.25   0.91
0.08:  0.34   0.88
0.10:  0.42   0.86
0.12:  0.48   0.84
0.14:  0.53   0.82
0.16:  0.58   0.80
0.18:  0.62   0.78
0.20:  0.65   0.77
0.22:  0.68   0.76
0.24:  0.71   0.75
0.26:  0.73   0.74
0.28:  0.75   0.73
0.30:  0.77   0.72
0.32:  0.78   0.71
0.34:  0.79   0.70
0.36:  0.81   0.69
0.38:  0.82   0.68
0.40:  0.83   0.68
0.42:  0.84   0.67
0.44:  0.84   0.66
0.46:  0.85   0.65
0.48:  0.86   0.64
0.50:  0.87   0.64
0.52:  0.88   0.63
0.54:  0.88   0.62
0.56:  0.89   0.61
0.58:  0.90   0.60
0.60:  0.90   0.59
0.62:  0.91   0.58
0.64:  0.91   0.58
0.66:  0.92   0.57
0.68:  0.92   0.56
{'fraction_non_alphanumeric': 0.11916349809885932, 'fraction_numerical': 0.053231939163498096, 'mean_word_length': 3.8474014006634722, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '147185', 'license': 'MIT', 'n_tokens_mistral': 5351, 'n_tokens_neox': 4971, 'n_words': 948, 'path': 'train-eval-sddEff.ipynb', 'repo_name': 'ncthuan/single-shot-detection-pytorch', 'size': '13150'}