<jupyter_start><jupyter_text>Dependencies<jupyter_code>!nvidia-smi
!jupyter notebook list
%env CUDA_VISIBLE_DEVICES=3
%matplotlib inline
%load_ext autoreload
%autoreload 2

import time
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms

from models import tiramisu
from models import tiramisu_bilinear
from models import tiramisu_m1
from models import unet
from models import linknet
from datasets import deepglobe
from datasets import maroads
from datasets import joint_transforms
import utils.imgs
import utils.training as train_utils

# tensorboard
from torch.utils.tensorboard import SummaryWriter<jupyter_output><empty_output><jupyter_text>DatasetDownload the DeepGlobe dataset from https://competitions.codalab.org/competitions/18467. Place it in datasets/deepglobe/dataset/train,test,validDownload the Massachusetts Road Dataset from https://www.cs.toronto.edu/~vmnih/data/. Combine the training, validation, and test sets, process with `crop_dataset.ipynb` and place the output in datasets/maroads/dataset/map,sat<jupyter_code>run = "expLink.4.dicebce"
DEEPGLOBE_PATH = Path('datasets/', 'deepglobe/dataset')
MAROADS_PATH = Path('datasets/', 'maroads/dataset')
RESULTS_PATH = Path('.results/')
WEIGHTS_PATH = Path('.weights/') / run
RUNS_PATH    = Path('.runs/')
RESULTS_PATH.mkdir(exist_ok=True)
WEIGHTS_PATH.mkdir(exist_ok=True)
RUNS_PATH.mkdir(exist_ok=True)

batch_size = 1 # TODO: Should be `MAX_BATCH_PER_CARD * torch.cuda.device_count()` (which in this case is 1 assuming max of 1 batch per card)
# resize = joint_transforms.JointRandomCrop((300, 300))

normalize = transforms.Normalize(mean=deepglobe.mean, std=deepglobe.std)
train_joint_transformer = transforms.Compose([
#     resize,
    joint_transforms.JointRandomHorizontalFlip(),
    joint_transforms.JointRandomVerticalFlip(),
    joint_transforms.JointRandomRotate()
    ])

train_slice = slice(None,4000)
test_slice = slice(4000,None)

train_dset = deepglobe.DeepGlobe(DEEPGLOBE_PATH, 'train', slc = train_slice,
    joint_transform=train_joint_transformer,
    transform=transforms.Compose([
        transforms.ColorJitter(brightness=.4,contrast=.4,saturation=.4),
        transforms.ToTensor(),
        normalize,
    ]))

train_dset_ma = maroads.MARoads(MAROADS_PATH, 
    joint_transform=train_joint_transformer,
    transform=transforms.Compose([
        transforms.ColorJitter(brightness=.4,contrast=.4,saturation=.4),
        transforms.ToTensor(),
        normalize,
    ]))

# print(len(train_dset_ma.imgs))
# print(len(train_dset_ma.msks))
train_dset_combine = torch.utils.data.ConcatDataset((train_dset, train_dset_ma))

# train_loader = torch.utils.data.DataLoader(train_dset, batch_size=batch_size, shuffle=True)
# train_loader = torch.utils.data.DataLoader(train_dset_ma, batch_size=batch_size, shuffle=True)
train_loader = torch.utils.data.DataLoader(
    train_dset_combine, batch_size=batch_size, shuffle=True)





# resize_joint_transformer = transforms.Compose([
#     resize
#     ])
resize_joint_transformer = None
val_dset = deepglobe.DeepGlobe(
    DEEPGLOBE_PATH, 'valid', joint_transform=resize_joint_transformer,
    transform=transforms.Compose([
        transforms.ToTensor(),
        normalize
    ]))
val_loader = torch.utils.data.DataLoader(
    val_dset, batch_size=batch_size, shuffle=False)

test_dset = deepglobe.DeepGlobe(
    DEEPGLOBE_PATH, 'train', joint_transform=resize_joint_transformer, slc = test_slice,
    transform=transforms.Compose([
        transforms.ToTensor(),
        normalize
    ]))
test_loader = torch.utils.data.DataLoader(
    test_dset, batch_size=batch_size, shuffle=False)
print("Train: %d" %len(train_loader.dataset))
print("Val: %d" %len(val_loader.dataset.imgs))
print("Test: %d" %len(test_loader.dataset.imgs))
# print("Classes: %d" % len(train_loader.dataset.classes))

print((iter(train_loader)))

inputs, targets = next(iter(train_loader))
print("Inputs: ", inputs.size())
print("Targets: ", targets.size())

utils.imgs.view_image(inputs[0])
# utils.imgs.view_image(targets[0])
utils.imgs.view_annotated(targets[0])

print(targets[0])<jupyter_output>Train: 4909
Val: 1243
Test: 2226
<torch.utils.data.dataloader._SingleProcessDataLoaderIter object at 0x7f7ba817a790>
Inputs:  torch.Size([1, 3, 1024, 1024])
Targets:  torch.Size([1, 1024, 1024])<jupyter_text>Train<jupyter_code>LR = 1e-4
LR_DECAY = 0.995
DECAY_EVERY_N_EPOCHS = 1
N_EPOCHS = 1000
torch.cuda.manual_seed(0)
from utils.bceloss import dice_bce_loss
from loss.BCESSIM import BCESSIM

model = linknet.LinkNet(n_classes=1).cuda()

optimizer = torch.optim.RMSprop(model.parameters(), lr=LR, weight_decay=1e-4)

criterion = dice_bce_loss()
# criterion = BCESSIM()

# summary(model, input_size=inputs[0].shape)
start_epoch = 0
start_epoch = train_utils.load_weights(model, (WEIGHTS_PATH/'latest.th'))
print("Starting from epoch", start_epoch)
# Writer will output to ./runs/ directory by default
writer = SummaryWriter(log_dir=(RUNS_PATH.as_posix() + "/" + "run" + str(run) + "/"))
#test_loader.dataset[0]
from torch.autograd import Variable

debug_max_size=None

# break # errors. Used to stop "run all"
for epoch in range(start_epoch, N_EPOCHS+1):
    since = time.time()

    ### Train ###
    trn_loss, trn_err = train_utils.train(
        model, train_loader, optimizer, criterion, epoch, debug_max_size=debug_max_size)
    print('Epoch {:d}\nTrain - Loss: {:.4f}, Acc: {:.4f}'.format(
        epoch, trn_loss, 1-trn_err))    
    time_elapsed = time.time() - since  
    print('Train Time {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    
    ### Test ###
    tes_loss, tes_err, tes_iou = train_utils.test(model, test_loader, criterion, epoch, debug_max_size=debug_max_size)    
    print('Tes - Loss: {:.4f} | Acc: {:.4f}'.format(tes_loss, 1-tes_err))
    time_elapsed = time.time() - since  
    print('Total Time {:.0f}m {:.0f}s\n'.format(
        time_elapsed // 60, time_elapsed % 60))
    
    val_loss = trn_loss
    val_err = trn_err
    
    ### Checkpoint ###    
    train_utils.save_weights(model, epoch, tes_loss, tes_err, WEIGHTS_PATH=WEIGHTS_PATH)

    ### Adjust Lr ###
#     train_utils.adjust_learning_rate(LR, LR_DECAY, optimizer, 
#                                      epoch, DECAY_EVERY_N_EPOCHS)
    
    # Log on tensorboard
    writer.add_scalar('Loss/train', trn_loss, epoch)
    writer.add_scalar('Loss/test', tes_loss, epoch)
    
    writer.add_scalar('Error/train', trn_err, epoch)
    writer.add_scalar('Error/test', tes_err, epoch)
    
    writer.add_scalar('Accuracy/train', 1-trn_err, epoch)
    writer.add_scalar('Accuracy/test', tes_iou, epoch)
    
#     writer.add_scalar('Accuracy/train', epoch_acc, epoch)
#     writer.add_scalar('Accuracy/test/noaug', do_valid(False), epoch)
#     writer.add_scalar('Accuracy/test/tta', do_valid(True), epoch)
    for param_group in optimizer.param_groups:
        writer.add_scalar('Params/learning_rage', param_group['lr'], epoch)
#     writer.add_scalar('params/learning_rate', optimizer.lr, epoch)
#     writer.add_scalar('Params/no_optim', no_optim, epoch)

    # log a sample image
    sample_images = [0,1030,281,623,636,655,1028,1353,2222,2224]
    for i in sample_images:
        inputs, targets, pred, loss, err, iou = train_utils.get_sample_predictions(model, test_loader, n=1, criterion=criterion, idx=i)
        raw = model(inputs.cuda()).cpu()        
        img = torchvision.utils.make_grid(torch.stack([
            inputs[0],
            targets[0].unsqueeze(0).expand(3,-1,-1).float(), 
            pred[0].unsqueeze(0).expand(3,-1,-1).float(),
            raw[0].expand(3,-1,-1).float()
        ]), normalize=True)
        
        writer.add_image('sample_pred/test/' + str(i), img, epoch)
    
    start_epoch = epoch<jupyter_output>torch.cuda.device_count() 1
multiplier 16.0
Epoch 42
Train - Loss: 0.9356, Acc: 0.6171
Train Time 54m 27s
Tes - Loss: 1.3120 | Acc: 0.2759
Total Time 63m 57s

torch.cuda.device_count() 1
multiplier 16.0
Epoch 43
Train - Loss: 0.9142, Acc: 0.6420
Train Time 48m 20s
Tes - Loss: 1.1390 | Acc: 0.4166
Total Time 57m 56s

torch.cuda.device_count() 1
multiplier 16.0<jupyter_text>Test<jupyter_code># returns test_loss, test_error, jaccard
train_utils.test(model, test_loader, criterion, epoch=1)
stats = train_utils.view_sample_predictions(model, test_loader, n=1, criterion=criterion)
print("loss", "error", "jaccard")
print(stats)

# !pip install torchsummary
from torchsummary import summary
summary(model, input_size=inputs[0].shape)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12530231486813315, 'fraction_numerical': 0.027409881377404122, 'mean_word_length': 4.66470971950424, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '82788', 'license': 'MIT', 'n_tokens_mistral': 3309, 'n_tokens_neox': 3087, 'n_words': 577, 'path': 'train_linknet.ipynb', 'repo_name': 'adriancampos/road-extraction', 'size': '8683'}