<jupyter_start><jupyter_text>Image网 Submission `128x128` This contains a submission for the Image网 leaderboard in the `128x128` category.In this notebook we:1. Train on 1 pretext task:   - Train a network to do image inpatining on Image网's `/train`, `/unsup` and `/val` images. 2. Train on 4 downstream tasks:  - We load the pretext weights and train for `5` epochs.  - We load the pretext weights and train for `20` epochs.  - We load the pretext weights and train for `80` epochs.  - We load the pretext weights and train for `200` epochs.  Our leaderboard submissions are the accuracies we get on each of the downstream tasks.<jupyter_code>import json
import torch

import numpy as np

from functools import partial

from fastai2.layers import Mish, MaxPool, LabelSmoothingCrossEntropy
from fastai2.learner import Learner
from fastai2.metrics import accuracy, top_k_accuracy
from fastai2.basics import DataBlock, RandomSplitter, GrandparentSplitter, CategoryBlock

from fastai2.optimizer import ranger, Adam, SGD, RMSProp

from fastai2.vision.all import *
from fastai2.vision.core import *
from fastai2.vision.augment import *
from fastai2.vision.learner import unet_learner, unet_config
from fastai2.vision.models.xresnet import xresnet50, xresnet34

from fastai2.data.transforms import Normalize, parent_label
from fastai2.data.external import download_url, URLs, untar_data


from fastcore.utils import num_cpus

from torch.nn import MSELoss
from torchvision.models import resnet34<jupyter_output><empty_output><jupyter_text>Pretext Task: Image Inpainting<jupyter_code># We create this dummy class in order to create a transform that ONLY operates on images of this type
# We will use it to create all input images
class PILImageInput(PILImage): pass

class RandomCutout(RandTransform):
    "Picks a random scaled crop of an image and resize it to `size`"
    split_idx = None
    def __init__(self, min_n_holes=5, max_n_holes=10, min_length=5, max_length=50, **kwargs):
        super().__init__(**kwargs)
        self.min_n_holes=min_n_holes
        self.max_n_holes=max_n_holes
        self.min_length=min_length
        self.max_length=max_length
        

    def encodes(self, x:PILImageInput):
        """
        Note that we're accepting our dummy PILImageInput class
        fastai2 will only pass images of this type to our encoder. 
        This means that our transform will only be applied to input images and won't
        be run against output images.
        """
        
        n_holes = np.random.randint(self.min_n_holes, self.max_n_holes)
        pixels = np.array(x) # Convert to mutable numpy array. FeelsBadMan
        h,w = pixels.shape[:2]

        for n in range(n_holes):
            h_length = np.random.randint(self.min_length, self.max_length)
            w_length = np.random.randint(self.min_length, self.max_length)
            h_y = np.random.randint(0, h)
            h_x = np.random.randint(0, w)
            y1 = int(np.clip(h_y - h_length / 2, 0, h))
            y2 = int(np.clip(h_y + h_length / 2, 0, h))
            x1 = int(np.clip(h_x - w_length / 2, 0, w))
            x2 = int(np.clip(h_x + w_length / 2, 0, w))
           
            pixels[y1:y2, x1:x2, :] = 0
            
        return Image.fromarray(pixels, mode='RGB')
torch.cuda.set_device(1)
# Default parameters
gpu=None
lr=1e-2
size=128
sqrmom=0.99
mom=0.9
eps=1e-6
epochs=15
bs=64
mixup=0.
opt='ranger',
arch='xresnet50'
sh=0.
sa=0
sym=0
beta=0.
act_fn='Mish'
fp16=0
pool='AvgPool',
dump=0
runs=1
meta=''
# Chosen parameters
lr=8e-3
sqrmom=0.99
mom=0.95
eps=1e-6
bs=64 
opt='ranger'
sa=1
fp16=1               #NOTE: My GPU cannot run fp16 :'(
arch='xresnet50' 
pool='MaxPool'

gpu=0

# NOTE: Normally loaded from their corresponding string
m = xresnet34
act_fn = Mish
pool = MaxPool
def get_dbunch(size, bs, sh=0., workers=None):
    if size<=224: 
        path = URLs.IMAGEWANG_160
    else: 
        path = URLs.IMAGEWANG
    source = untar_data(path)
    if workers is None: workers = min(8, num_cpus())
    #CHANGE: Input is ImageBlock(cls=PILImageInput)
    #CHANGE: Output is ImageBlock
    #CHANGE: Splitter is RandomSplitter (instead of on /val folder)
    
    item_tfms=[RandomResizedCrop(size, min_scale=0.35), FlipItem(0.5), RandomCutout]
#     batch_tfms=RandomErasing(p=0.9, max_count=3, sh=sh) if sh else None
    batch_tfms = [Normalize.from_stats(*imagenet_stats)]
    
    dblock = DataBlock(blocks=(ImageBlock(cls=PILImageInput), ImageBlock),
                       splitter=RandomSplitter(0.1),
                       get_items=get_image_files, 
                       get_y=lambda o: o,
                      item_tfms=item_tfms,
                      batch_tfms=batch_tfms)
    return dblock.dataloaders(source, path=source, bs=bs, num_workers=workers)
name = 'imagewang_inpainting_80_epochs_nopretrain_normalized.pth'
# Use the Ranger optimizer
opt_func = partial(ranger, mom=mom, sqr_mom=sqrmom, eps=eps, beta=beta)
size = 128
bs = 64

dbunch = get_dbunch(size, bs, sh=sh)
#CHANGE: We're predicting pixel values, so we're just going to predict an output for each RGB channel
dbunch.vocab = ['R', 'G', 'B']
len(dbunch.train.dataset), len(dbunch.valid.dataset)
dbunch.show_batch()
learn = unet_learner(dbunch, partial(m, sa=sa), pretrained=False, opt_func=opt_func, metrics=[], loss_func=MSELoss()).to_fp16()
cbs = MixUp(mixup) if mixup else []
learn.fit_flat_cos(80, lr, wd=1e-2, cbs=cbs)



# I'm not using fastai2's .export() because I only want to save 
# the model's parameters. 
torch.save(learn.model[0].state_dict(), name)<jupyter_output><empty_output><jupyter_text>Downstream Task: Image Classification<jupyter_code>def get_dbunch(size, bs, sh=0., workers=None):
    if size<=224: 
        path = URLs.IMAGEWANG_160
    else: 
        path = URLs.IMAGEWANG
    source = untar_data(path)
    if workers is None: workers = min(8, num_cpus())
    item_tfms=[RandomResizedCrop(size, min_scale=0.35), FlipItem(0.5)]
    batch_tfms = [Normalize.from_stats(*imagenet_stats)]
#     batch_tfms=RandomErasing(p=0.9, max_count=3, sh=sh) if sh else None
    dblock = DataBlock(blocks=(ImageBlock, CategoryBlock),
                       splitter=GrandparentSplitter(valid_name='val'),
                       get_items=get_image_files, get_y=parent_label,
                      item_tfms=item_tfms, batch_tfms=batch_tfms)
    return dblock.dataloaders(source, path=source, bs=bs, num_workers=workers,
                             )#item_tfms=item_tfms, batch_tfms=batch_tfms)
dbunch = get_dbunch(size, bs, sh=sh)
m_part = partial(m, c_out=20, act_cls=torch.nn.ReLU, sa=sa, sym=sym, pool=pool)<jupyter_output><empty_output><jupyter_text>5 Epochs<jupyter_code>epochs = 5
runs = 5
for run in range(runs):
        print(f'Run: {run}')
                        
        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 20))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func, pretrained=False,
                metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy(),
                config={'custom_head':ch})
        
        if dump: print(learn.model); exit()
        if fp16: learn = learn.to_fp16()
        cbs = MixUp(mixup) if mixup else []
            
#         # Load weights generated from training on our pretext task
        state_dict = torch.load(name)
        learn.model[0].load_state_dict(state_dict)
        
        learn.unfreeze()
        learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)<jupyter_output>Run: 0<jupyter_text>* Run 1: 0.362942* Run 2: 0.372868* Run 3: 0.342326* Run 4: 0.360143* Run 5: 0.357088Accuracy: **35.91%**  20 Epochs<jupyter_code>epochs = 20
runs = 3
for run in range(runs):
        print(f'Run: {run}')
                        
        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 20))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func, pretrained=False,
                metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy(),
                config={'custom_head':ch})
        
        if dump: print(learn.model); exit()
        if fp16: learn = learn.to_fp16()
        cbs = MixUp(mixup) if mixup else []
            
#         # Load weights generated from training on our pretext task
        state_dict = torch.load(name)
        learn.model[0].load_state_dict(state_dict)
        
        learn.unfreeze()
        learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)<jupyter_output>Run: 0<jupyter_text>* Run 1: 0.592263* Run 2: 0.588445* Run 3: 0.595571Accuracy: **59.21%**  80 epochs<jupyter_code>epochs = 80
runs = 1
for run in range(runs):
        print(f'Run: {run}')
                        
        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 20))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func, pretrained=False,
                metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy(),
                config={'custom_head':ch})
        
        if dump: print(learn.model); exit()
        if fp16: learn = learn.to_fp16()
        cbs = MixUp(mixup) if mixup else []
            
#         # Load weights generated from training on our pretext task
        state_dict = torch.load(name)
        learn.model[0].load_state_dict(state_dict)
        
        learn.unfreeze()
        learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)<jupyter_output><empty_output><jupyter_text>Accuracy: **61.44%**  200 epochs<jupyter_code>epochs = 200
runs = 1
for run in range(runs):
        print(f'Run: {run}')
                        
        ch = nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(512, 20))
        learn = cnn_learner(dbunch, m_part, opt_func=opt_func, pretrained=False,
                metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy(),
                config={'custom_head':ch})
        
        if dump: print(learn.model); exit()
        if fp16: learn = learn.to_fp16()
        cbs = MixUp(mixup) if mixup else []
            
#         # Load weights generated from training on our pretext task
        state_dict = torch.load('imagewang_inpainting_15_epochs_nopretrain.pth')
        learn.model[0].load_state_dict(state_dict)
        
        learn.unfreeze()
        learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1082623839009288, 'fraction_numerical': 0.03173374613003096, 'mean_word_length': 3.097106619104241, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '44310', 'license': 'Apache-2.0', 'n_tokens_mistral': 3837, 'n_tokens_neox': 3592, 'n_words': 847, 'path': '00_ImageWang_Inpainting_baseline_ep80.ipynb', 'repo_name': 'WAMRI-AI/imagewang', 'size': '10336'}