<jupyter_start><jupyter_text>- https://www.kaggle.com/abhishek/very-simple-pytorch-training-0-59?scriptVersionId=16436961- https://www.kaggle.com/abhishek/pytorch-inference-kernel-lazy-tta<jupyter_code>dbg = True
if dbg:
    dbgtrnsz=100
    dbgvalsz=100
PRFX = 'devCv0630'
SEED = 111
SZ = (224, 224)
BSZ = 16
BSZ_INFER = 32
N_EPOCHS = 2<jupyter_output><empty_output><jupyter_text>setup<jupyter_code>import random 
import numpy as np
import torch
import os
import datetime

def set_torch_seed(seed=SEED):
    os.environ['PYTHONHASHSEED'] = str(seed)
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    
    if torch.cuda.is_available(): 
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed) 
        torch.backends.cudnn.deterministic = True 
        torch.backends.cudnn.benchmark = False

set_torch_seed()
import pandas as pd
from collections import Counter
import time
from tqdm import tqdm_notebook as tqdm


from torch.utils.data import Dataset
import torch
import torchvision
import torch.nn as nn
import torch.optim as optim
from torchvision import transforms
from torch.optim import lr_scheduler

from PIL import Image, ImageFile

ImageFile.LOAD_TRUNCATED_IMAGES = True


from pathlib import Path
p_o = f'../output/{PRFX}'
Path(p_o).mkdir(exist_ok=True, parents=True)<jupyter_output><empty_output><jupyter_text>preprocess<jupyter_code>img2grd = []
p = '../input/aptos2019-blindness-detection'
pp = Path(p)
train = pd.read_csv(pp/'train.csv')

len_blnd = len(train)

img2grd_blnd = [(f'{p}/train_images/{o[0]}.png',o[1])  for o in train.values]

img2grd += img2grd_blnd
display(len(img2grd))
display(Counter(o[1] for o in img2grd).most_common())
p = '../input/diabetic-retinopathy-detection'
pp = Path(p)
train=pd.read_csv(pp/'trainLabels.csv')
test=pd.read_csv(pp/'retinopathy_solution.csv')

img2grd_diab_train=[(f'{p}/train_images/{o[0]}.jpeg',o[1])  for o in train.values]
img2grd_diab_test=[(f'{p}/test_images/{o[0]}.jpeg',o[1])  for o in test.values]
img2grd += img2grd_diab_train
display(len(img2grd))
display(Counter(o[1] for o in img2grd).most_common())
img2grd += img2grd_diab_test
len(img2grd)
display(Counter(o[1] for o in img2grd).most_common())
p = '../input/IDRID/B. Disease Grading'
pp = Path(p)
train=pd.read_csv(pp/'2. Groundtruths/a. IDRiD_Disease Grading_Training Labels.csv')
test=pd.read_csv(pp/'2. Groundtruths/b. IDRiD_Disease Grading_Testing Labels.csv')

img2grd_idrid_train=[(f'{p}/1. Original Images/a. Training Set/{o[0]}.jpg',o[1])  for o in train.values]
img2grd_idrid_test=[(f'{p}/1. Original Images/b. Testing Set/{o[0]}.jpg',o[1])  for o in test.values]
img2grd += img2grd_idrid_train
display(len(img2grd))
display(Counter(o[1] for o in img2grd).most_common())
img2grd += img2grd_idrid_test
len(img2grd)
display(Counter(o[1] for o in img2grd).most_common())
img2grd = np.array(img2grd)
if np.all([Path(o[0]).exists() for o in img2grd]): print('All files are here!')<jupyter_output>All files are here!<jupyter_text>dataset<jupyter_code>set_torch_seed()
idx_val = range(len_blnd)
idx_trn = range(len_blnd, len(img2grd))

img2grd_trn = img2grd[idx_trn]
img2grd_val = img2grd[idx_val]

display(len(img2grd_trn), len(img2grd_val))

img2grd_trn[:3], img2grd_val[:3]
if dbg:
    img2grd_trn = img2grd_trn[:dbgtrnsz]
    img2grd_val = img2grd_val[:dbgvalsz]
class BlndDataset(Dataset):
    def __init__(self, img2grd, transform):
        self.img2grd = img2grd
        self.transform = transform

    def __len__(self):
        return len(self.img2grd)

    def __getitem__(self, idx):
        img,grd = img2grd[idx]
        image = self.transform(Image.open(img))
        label = torch.tensor(int(grd))
        return image, label

transform_train = transforms.Compose([
    transforms.Resize(SZ),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])

ds_trn = BlndDataset(img2grd_trn, transform=transform_train)
ds_val = BlndDataset(img2grd_val, transform=transform_train)

data_loader = torch.utils.data.DataLoader(ds_trn, batch_size=BSZ, shuffle=True, num_workers=4)
data_loader_val = torch.utils.data.DataLoader(ds_val, batch_size=BSZ_INFER, shuffle=False, num_workers=4)<jupyter_output><empty_output><jupyter_text>model<jupyter_code>model = torchvision.models.resnet50(pretrained=False)
model.load_state_dict(torch.load("../input/pytorch_models/resnet50-19c8e357.pth"));<jupyter_output><empty_output><jupyter_text>model<jupyter_code># model.fc = nn.Linear(2048, 1)
model.fc = nn.Sequential(
                          nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
                          nn.Dropout(p=0.25),
                          nn.Linear(in_features=2048, out_features=2048, bias=True),
                          nn.ReLU(),
                          nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
                          nn.Dropout(p=0.5),
                          nn.Linear(in_features=2048, out_features=1, bias=True),
                         )

device = torch.device("cuda")

model = model.to(device)<jupyter_output><empty_output><jupyter_text>model<jupyter_code>plist = [
         {'params': model.layer4.parameters(), 'lr': 1e-4, 'weight': 0.001},
         {'params': model.fc.parameters(), 'lr': 1e-3}
         ]

optimizer = optim.Adam(plist, lr=0.001)
scheduler = lr_scheduler.StepLR(optimizer, step_size=10)<jupyter_output><empty_output><jupyter_text>Training Loop<jupyter_code>data_loader
set_torch_seed()
len_dl = len(data_loader)
since = time.time()
criterion = nn.MSELoss()
for epoch in range(N_EPOCHS):
    print(f'Epoch {epoch}/{N_EPOCHS-1}')
    scheduler.step()
    model.train()
    running_loss = 0.0
    for step, d in enumerate(data_loader):
        inputs = d[0]
        labels = d[1].view(-1, 1)
        inputs = inputs.to(device, dtype=torch.float)
        labels = labels.to(device, dtype=torch.float)
        optimizer.zero_grad()
        with torch.set_grad_enabled(True):
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
        running_loss += loss.item() * inputs.size(0)
        if (step+1) % (2 if dbg else 1000) == 0: 
            print(f'[{datetime.datetime.now()}] epoch-{epoch} step-{step+1}/{len_dl} loss: {running_loss/step:.5f}')
    epoch_loss = running_loss / len_dl
    
#     model.eval()
    
#     test_preds1 = np.zeros((len(data_loader_val), 1))
#     for step, d in enumerate(data_loader_val):
#         inputs = d[0]
#         labels = d[1].view(-1, 1)
#         inputs = inputs.to(device, dtype=torch.float)
#         labels = labels.to(device, dtype=torch.float)
#         with torch.no_grad():
#             outputs = model(inputs)
#             loss = criterion(outputs, labels)
        
    
    
    print(f'Training Loss: {epoch_loss:.4f}')

time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
torch.save(model.state_dict(), f"{p_o}/model.bin")<jupyter_output>Epoch 0/1
[2019-07-01 02:58:18.503752] epoch-0 step-2/7 loss: 12.25622
[2019-07-01 02:58:18.913291] epoch-0 step-4/7 loss: 10.12270
[2019-07-01 02:58:20.814748] epoch-0 step-6/7 loss: 8.06137
Training Loss: 5.7759
Epoch 1/1
[2019-07-01 02:58:23.402183] epoch-1 step-2/7 loss: 7.75631
[2019-07-01 02:58:23.859841] epoch-1 step-4/7 loss: 4.65631
[2019-07-01 02:58:25.693556] epoch-1 step-6/7 loss: 4.35087
Training Loss: 3.2589
Training complete in 0m 10s
{'fraction_non_alphanumeric': 0.1427255985267035, 'fraction_numerical': 0.056169429097605895, 'mean_word_length': 3.8925353925353927, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '56466', 'license': 'Apache-2.0', 'n_tokens_mistral': 3218, 'n_tokens_neox': 2977, 'n_words': 496, 'path': 'nbs/devCv0630-deving_trn_runs.ipynb', 'repo_name': 'yang-zhang/aptos2019-blindness-detection', 'size': '7602'}