<jupyter_start><jupyter_text>Image网 Submission `128x128` This contains a submission for the Image网 leaderboard in the `128x128` category.In this notebook we:1. Train on 1 pretext task:   - Train a network to do image inpatining on Image网's `/train`, `/unsup` and `/val` images. 2. Train on 4 downstream tasks:  - We load the pretext weights and train for `5` epochs.  - We load the pretext weights and train for `20` epochs.  - We load the pretext weights and train for `80` epochs.  - We load the pretext weights and train for `200` epochs.  Our leaderboard submissions are the accuracies we get on each of the downstream tasks.<jupyter_code>import json
import torch

import numpy as np

from functools import partial

from fastai2.layers import Mish, MaxPool, LabelSmoothingCrossEntropy
from fastai2.learner import Learner
from fastai2.metrics import accuracy, top_k_accuracy
from fastai2.basics import DataBlock, RandomSplitter, GrandparentSplitter, CategoryBlock

from fastai2.optimizer import ranger, Adam, SGD, RMSProp

from fastai2.vision.all import *
from fastai2.vision.core import *
from fastai2.vision.augment import *
from fastai2.vision.learner import unet_learner, unet_config
from fastai2.vision.models.xresnet import xresnet50, xresnet34

from fastai2.data.transforms import Normalize, parent_label
from fastai2.data.external import download_url, URLs, untar_data


from fastcore.utils import num_cpus

from torch.nn import MSELoss
from torchvision.models import resnet34
torch.cuda.set_device(1)<jupyter_output><empty_output><jupyter_text>Pretext Task: Image Inpainting<jupyter_code># Chosen parameters
lr=8e-3
sqrmom=0.99
mom=0.95
beta=0.
eps=1e-4
bs=64 
opt='ranger'
sa=1
fp16=1
pool='MaxPool'
sym=0

# NOTE: Normally loaded from their corresponding string
m = xresnet34
act_fn = Mish
pool = MaxPool

nc=20
source = untar_data(URLs.IMAGEWANG_160)
len(get_image_files(source/'unsup')), len(get_image_files(source/'train'))
# [s.name for s in (source/'train').ls()], [s.name for s in (source/'val').ls()]
def get_dbunch(size, bs, workers=8):
    path = URLs.IMAGEWANG_160 if size <= 160 else URLs.IMAGEWANG
    source = untar_data(path)
    
    files = get_image_files(source)
    
    tfms = [[PILImage.create, ToTensor, Resize(size)], 
            [lambda x: x.parent.name, Categorize()]]
    
    dsets = Datasets(files, tfms=tfms, splits=RandomSplitter(valid_pct=0.1)(files))
    
    batch_tfms = [IntToFloatTensor, Normalize.from_stats(*imagenet_stats)]
    dls = dsets.dataloaders(bs=bs, num_workers=workers, after_batch=batch_tfms)
    dls.path = source
    return dls
# Use the Ranger optimizer
opt_func = partial(ranger, mom=mom, sqr_mom=sqrmom, eps=eps, beta=beta)
size = 128
bs = 256

dbunch = get_dbunch(160, bs)
dbunch.c = nc
len(dbunch.train.dataset)
xb,yb = dbunch.one_batch()
xb.shape, yb.shape
dbunch.show_batch()

from rsna_retro.self_supervised import *
from rsna_retro.contrastive_loss import *
#export
def get_aug_pipe(size, min_scale=0.4, stats=(mean,std), **kwargs):
    tfms = [Normalize.from_stats(*stats)] + aug_transforms(size=size, min_scale=min_scale, **kwargs)
    return Pipeline(tfms)
m_part = partial(m, c_out=nc, act_cls=torch.nn.ReLU, sa=sa, sym=sym, pool=pool)
ss_name = 'imagewang_contrast_ss4'
aug = get_aug_pipe(size, min_scale=0.20, mult=1, stats=imagenet_stats)
aug2 = get_aug_pipe(size, min_scale=0.15, mult=2, stats=imagenet_stats)
cbs = SSCallback(XentLoss(0.1), size=size, aug_targ=aug, aug_pos=aug2, multi_loss=False)
# cbs = SSCallback(BatchContrastiveLoss(XentLoss(0.1)), size=size, aug_targ=aug, aug_pos=aug2, multi_loss=False)


# ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 128))
ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 256), nn.ReLU(), nn.Linear(256, 64))
learn = cnn_learner(dbunch, m_part, opt_func=opt_func,
                    metrics=[], loss_func=CrossEntropyLossFlat(), cbs=cbs,
                    config={'custom_head':ch}, pretrained=False,
                   )
#         metrics=[], loss_func=LabelSmoothingCrossEntropy())
learn.unfreeze()
learn.fit_flat_cos(30, 1e-2, wd=1e-3)
learn.save(f'{ss_name}')
learn.load(f'{ss_name}')
learn.unfreeze()
learn.fit_flat_cos(30, 4e-3, wd=1e-3)
learn.save(f'{ss_name}-1')<jupyter_output><empty_output><jupyter_text>Downstream Task: Image Classification<jupyter_code>def get_dbunch(size, bs, workers=8):
    path = URLs.IMAGEWANG_160 if size <= 160 else URLs.IMAGEWANG
    source = untar_data(path)
    
    files = get_image_files(source)
    
    item_aug = [RandomResizedCrop(size, min_scale=0.35), FlipItem(0.5)]
    tfms = [[PILImage.create, ToTensor, *item_aug], 
            [lambda x: x.parent.name, Categorize()]]
    
    dsets = Datasets(files, tfms=tfms, splits=GrandparentSplitter(valid_name='val')(files))
    
    batch_tfms = [IntToFloatTensor, Normalize.from_stats(*imagenet_stats)]
    dls = dsets.dataloaders(bs=bs, num_workers=workers, after_batch=batch_tfms)
    dls.path = source
    return dls
size=128
bs = 64
dbunch = get_dbunch(size, bs)
dbunch.show_batch()<jupyter_output><empty_output><jupyter_text>5 Epochs<jupyter_code>epochs = 5
runs = 1
for run in range(runs):
        print(f'Run: {run}')
        
#         aug = get_aug_pipe(size, min_scale=1.0, mult=1)
#         aug2 = get_aug_pipe(size, min_scale=0.3, mult=1)
#         cbs = SSCallback(BatchContrastiveLoss(XentContrastiveLoss(0.05)), size=size, aug_targ=aug, aug_pos=aug2, multi_loss=True)

        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 256))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func,
                metrics=[accuracy,top_k_accuracy], loss_func=CrossEntropyLossFlat(),
                pretrained=False,
                config={'custom_head':ch})#, cbs=cbs)
        
#         if fp16: learn = learn.to_fp16()
        learn.load(f'{ss_name}-1', strict=False)
        learn.model[1][-1] = nn.Linear(512, 20)
        
        learn.unfreeze()
#         learn.freeze()
#         learn.lr_find()
        learn.fit_flat_cos(epochs, 1e-2, wd=1e-3)
for run in range(runs):
        print(f'Run: {run}')
        
        aug = get_aug_pipe(size, min_scale=0.35, mult=1)
        aug2 = get_aug_pipe(size, min_scale=0.3, mult=1)
        cbs = SSCallback(XentLoss(0.1), size=size, aug_targ=aug, aug_pos=aug2, multi_loss=True)

        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 256))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func,
                metrics=[accuracy,top_k_accuracy], loss_func=CrossEntropyLossFlat(),
                pretrained=True,
                config={'custom_head':ch}, cbs=cbs)
        
#         if fp16: learn = learn.to_fp16()
#         learn.load(f'{ss_name}-1', strict=False)
        learn.model[1][-1] = nn.Linear(512, 20)
        
        learn.unfreeze()
#         learn.freeze()
#         learn.lr_find()
        learn.fit_flat_cos(epochs, 2e-2, wd=1e-2)
len(learn.opt.opt.param_groups)<jupyter_output><empty_output><jupyter_text>- Run 1: 0.403156- Run 2: 0.404429- Run 3: 0.416645- Run 4: 0.407228- Run 5: 0.412064Average: **40.87%**  20 Epochs<jupyter_code>epochs = 20
runs = 3
for run in range(runs):
        print(f'Run: {run}')
        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 20))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func,
                metrics=[accuracy,top_k_accuracy], loss_func=CrossEntropyLossFlat(),
                config={'custom_head':ch})#, cbs=cbs)
        
        if dump: print(learn.model); exit()
#         if fp16: learn = learn.to_fp16()
        cbs = MixUp(mixup) if mixup else []
        learn.load(ss_name, strict=True)
        
        learn.freeze()
        learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)<jupyter_output><empty_output><jupyter_text>- Run 1: 0.610333- Run 2: 0.618733- Run 3: 0.605498Average: **61.15%**  80 epochs<jupyter_code>epochs = 80
runs = 1
for run in range(runs):
        print(f'Run: {run}')
        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 20))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func,
                metrics=[accuracy,top_k_accuracy], loss_func=CrossEntropyLossFlat(),
                config={'custom_head':ch})#, cbs=cbs)
        
        if dump: print(learn.model); exit()
#         if fp16: learn = learn.to_fp16()
        cbs = MixUp(mixup) if mixup else []
        learn.load(ss_name, strict=True)
        
        learn.freeze()
        learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)<jupyter_output><empty_output><jupyter_text>Accuracy: **62.18%**  200 epochs<jupyter_code>epochs = 200
runs = 1
for run in range(runs):
        print(f'Run: {run}')
        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 20))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func,
                metrics=[accuracy,top_k_accuracy], loss_func=CrossEntropyLossFlat(),
                config={'custom_head':ch})#, cbs=cbs)
        
        if dump: print(learn.model); exit()
#         if fp16: learn = learn.to_fp16()
        cbs = MixUp(mixup) if mixup else []
        learn.load(ss_name, strict=True)
        
        learn.freeze()
        learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1302240413614821, 'fraction_numerical': 0.03877638948728996, 'mean_word_length': 3.734829168791433, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '113744', 'license': 'Apache-2.0', 'n_tokens_mistral': 3721, 'n_tokens_neox': 3474, 'n_words': 685, 'path': '09_ImageWang_Leadboard_SS_5.ipynb', 'repo_name': 'bearpelican/rsna_retro', 'size': '9284'}