<jupyter_start><jupyter_text>Pytorch + GPU + Inference > Pytorch🔥 PyTroch - Lightning⚡️ GPU⏱This notebook explore the training experiment under pytorch lightning framework  Dependencies Install high-level frameworks* EfficientNet PyTorch* PyTorch Lightning<jupyter_code># !pip install efficientnet_pytorch
# !pip install pytorch-lightning
# !pip install forgebox
import albumentations
import os
import gc
import torch


import numpy as np
import pandas as pd

import torch.nn as nn
from sklearn import metrics
from sklearn import model_selection
from torch.nn import functional as F
from torch.utils.data import Dataset,DataLoader

import efficientnet_pytorch

from pytorch_lightning import LightningModule,Trainer
from pytorch_lightning.loggers import TensorBoardLogger
from pytorch_lightning.callbacks import Callback,ModelCheckpoint,EarlyStopping,ProgressBar
from pytorch_lightning.metrics import Accuracy,F1,AUC

from sklearn.metrics import auc,roc_auc_score,accuracy_score

from PIL import Image
from pathlib import Path
from tqdm import trange
from forgebox.imports import *
from torchvision import transforms as trf
MEAN=[0.485, 0.456, 0.406]
STD=[0.229, 0.224, 0.225] 

# MEAN = [0.80619959, 0.62115946, 0.59133584]
# STD = [0.15061945, 0.17709774, 0.20317172]
def get_aug(train = False):
    mean = MEAN
    std = STD
    
    if train:
        train_aug = albumentations.Compose(
        [albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),
        albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),
            albumentations.Flip(p=0.5)])
        return train_aug
    
    else:
        valid_aug = albumentations.Compose(
        [albumentations.Normalize(mean, std, max_pixel_value=255.0,always_apply=True)])
        return valid_aug
aug = get_aug(train=True)
!ls /GCI/bioasq/pdl1/model/fold_0
KFOLD = 5

ON_KERNEL = False
if ON_KERNEL:
    # on kernel
    DATA = "../input/siim-isic-melanoma-classification/"
    TRAIN_CSV = "../input/siim-isic-melanoma-classification/train.csv"
    TEST_CSV = "../input/siim-isic-melanoma-classification/test.csv"
    TRAIN_DIR = Path("../input/gastly-detailed-512x-4shots/train_data/img/train")
    TEST_DIR = Path("../input/gastly-detailed-512x-4shots/test_data/img/test")
else:
    # on server
    DATA = Path("/GCI/bioasq/pdl1/")
    TRAIN_CSV = DATA/"csv"/"train.csv"
    TEST_CSV = DATA/"csv"/"test.csv"
    TRAIN_DIR = DATA/"train_data/img/train"
    TEST_DIR = DATA/"test_data/img/test"
from glob import glob
def model_folds(path,mode:str = "last"):
    path = Path(path)
    folds = list(p for p in path.ls() if "fold" in p)
    kw = "last" if mode=="last" else "epoch"
    result = []
    for f in folds:
        for ckpt in (path/f).ls():
            if ckpt[-4:]=="ckpt" and kw in ckpt: 
                result.append(str(path/f/ckpt))
    return result
WEIGHTS = model_folds("/GCI/bioasq/pdl1/model/",mode = "best")
class EfficientNet(LightningModule):
    def __init__(self,tag='efficientnet-b0',batch_size=16):
        super(EfficientNet, self).__init__()
        self.batch_size = batch_size
        base_model = efficientnet_pytorch\
            .EfficientNet\
            .from_pretrained(tag)
            
        base_model._fc = nn.Linear(
            in_features={"efficientnet-b0":1280, 
                         "efficientnet-b5":2048,}[tag],
            out_features=1, 
            bias=True
        )
        self.base_model = base_model
        self.acc = Accuracy(num_classes=2)
        self.f1 = F1(num_classes=2)
        self.sigmoid = nn.Sigmoid()
        
        self.crit = nn.BCEWithLogitsLoss()
        
    def p10(self,logits):
        return (self.sigmoid(logits)>.5).float()
        
    def forward(self, image):
        out = self.base_model(image)
        return out
    
    @classmethod
    def from_fold(cls,tag,fold,weight_path=WEIGHTS):
        path = list(p for p in weight_path if f"fold_{fold}" in p)[0]
        print(f"Loading fold {fold} with path:\n{path}")
        model = cls(tag)
        model.load_state_dict(torch.load(path,map_location='cpu')['state_dict'])
        model.fold = fold
        model = model.eval()
        return model
model = EfficientNet.from_fold(tag = 'efficientnet-b5',fold = 0)
folds_csv = pd.read_csv("train_folds.csv")<jupyter_output><empty_output><jupyter_text>Getting label by folds<jupyter_code>def get_label(fold):
    df = folds_csv[folds_csv.kfold==fold]
    df = df[~df.duplicated(subset=["image_name",])].reset_index(drop=True)[["image_name","target"]]
    return df

def mean_std(t):
    return {"mean":t.mean(),"std":t.std()}
get_label(2)
def create_dl(f):
    def wrapper(*args,**kwargs):
        ds = f(*args,**kwargs)
        if kwargs.get("return_dl"):
            return DataLoader(ds, batch_size = 128,shuffle = False, num_workers = 8)
        else:
            return ds
    return wrapper

class siimData(Dataset):
    def __init__(self,df,path,aug):
        self.df = df
        self.indices = df.index
        self.open_rgb = self.open_function(Path(path))
        self.aug = aug
        
    def __len__(self,):return len(self.df)
        
    def __getitem__(self,idx):
        row = dict(self.df.loc[self.indices[idx]])
        image_name = row["image_name"]
        target = row["target"] if "target" in row else -1
        img = self.open_rgb(image_name)
        
        arr = self.aug(image = np.array(img))["image"]
        return np.moveaxis(arr,[0,1,2],[1,2,0]), target, image_name
    
    @staticmethod
    def open_function(parent:Path, mode:str = "RGB", resize:int = None,format_ = "jpg"):
        def open_image(image_id:str):
            img = Image.open(parent/f"{image_id}.{format_}").convert(mode)
            if resize:
                img = img.resize((resize,resize))
            return img
        return open_image
    
    @classmethod
    @create_dl
    def infer_folds(cls,fold:int,return_dl = True):
        """
        Dataset for inference on a fold's validation data
        """
        return siimData(get_label(fold),TRAIN_DIR,get_aug(train=False))
    
    @classmethod
    @create_dl
    def infer_test(cls,return_dl = True):
        """
        Dataset for inference on test dataset
        """
        return cls(pd.read_csv(TEST_CSV),TEST_DIR,get_aug(train=False))
siimData.infer_test(),siimData.infer_test(return_dl = True)
siimData.infer_folds(2,return_dl=True)
from tqdm.notebook import tqdm_notebook as tqdm
from itertools import chain
from forgebox.ftorch.cuda import CudaHandler
def tonp(x): return x.cpu().numpy()
def pred(model,dl,device="cpu"):
    model = model.to(device)
    gen = iter(dl)
    Names,Y,Y_ = [],[],[]
    with torch.no_grad():
        for x,y,names in tqdm(dl):
            x = x.to(device)
            y = y.to(device)
            Names.append(names)
            y_ = model.sigmoid(model(x))
            Y.append(tonp(y).reshape(-1))
            Y_.append(tonp(y_).reshape(-1))
    Names = list(chain(*Names))
    Y = np.concatenate(Y)
    Y_ = np.concatenate(Y_)
    
    model = model.to("cpu")
    return pd.DataFrame({"image_name":Names,"target":Y_,"label":Y})
idle = CudaHandler().idle().device
for fold in range(KFOLD):
    pred_df = pred(EfficientNet.from_fold(tag = 'efficientnet-b5',fold = fold),
                   siimData.infer_folds(fold,
                                        return_dl=True),
                   device = idle
                  )
    auc = roc_auc_score(pred_df.label,pred_df.target,)
    acc = accuracy_score(pred_df.label,pred_df.target>0.5)
    print(f"[FOLD {fold}]>>> AUC:\t{auc}\nACC:\t{acc}")<jupyter_output>Loading fold 0 with path:
/GCI/bioasq/pdl1/model/fold_0/epoch=2.ckpt
Loaded pretrained weights for efficientnet-b5<jupyter_text>[FOLD 0]>>> AUC:	0.8798783538374872ACC:	0.9817386054935104[FOLD 1]>>> AUC:	0.8418436541446592ACC:	0.979622641509434[FOLD 2]>>> AUC:	0.8823814476856886ACC:	0.9802264150943396[FOLD 3]>>> AUC:	0.9093121417952396ACC:	0.9829433962264151[FOLD 4]>>> AUC:	0.8747655745197231ACC:	0.9803773584905661<jupyter_code>y<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12580326248146317, 'fraction_numerical': 0.041151754819574886, 'mean_word_length': 3.0525788683024535, 'pattern_counts': {'":': 7, '<': 12, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '657978', 'license': 'Apache-2.0', 'n_tokens_mistral': 3074, 'n_tokens_neox': 2867, 'n_words': 535, 'path': 'nbs/predict_inference.ipynb', 'repo_name': 'iofthetiger/siim', 'size': '8092'}