<jupyter_start><jupyter_text>Inpainting with Variable Dataset Size Recall that the Image网 dataset consists of:1. A `/val` folder with 10 classes.2. A `/train` folder with 20 classes.   - There are ~125 images in each class that exists in `/val`. There are   - There are ~1,300 images in each class that does not exist in `/val`3. An `/unsup` folder with 7,750 unlabelled images. The question we would like to answer with this notebook is:> What is the effect of dataset size during pretext training on downstream task performance?To answer this question we will consider four different datasets, each built from ImageWang.They are:1. All data in `/train`, `/unsup` and `/val`2. All data in `/train`, `/unsup`3. All data in `/train`4. Only Data in `/train` that has a corresponding class in `/val`<jupyter_code>import gc
import json
import torch
import numpy as np

from config import config
from RandomCutout import RandomCutout, PILImageInput

from fastai2.basics import *
from fastai2.vision.all import *

from torch.nn import MSELoss
from functools import partial
# Default parameters
lr=config['lr']
size=config['size']
sqrmom=config['sqrmom']
mom=config['mom']
eps=config['eps']
epochs=config['epochs']
bs=config['bs']
opt=config['opt']
sh=config['sh']
sa=config['sa']
sym=config['sym']
beta=config['beta']
act_fn=config['act_fn']
fp16=config['fp16']
pool=config['pool']
runs=config['runs']

model = config['model']

if   opt=='adam'  : opt_func = partial(Adam, mom=mom, sqr_mom=sqrmom, eps=eps)
elif opt=='rms'   : opt_func = partial(RMSProp, sqr_mom=sqrmom)
elif opt=='sgd'   : opt_func = partial(SGD, mom=mom)
elif opt=='ranger': opt_func = partial(ranger, mom=mom, sqr_mom=sqrmom, eps=eps, beta=beta)
    
size = 128
bs = 64
runs=3
print("lr", lr)
print("size", size)
print("sqrmom", sqrmom)
print("mom", mom)
print("eps", eps)
print("epochs", epochs)
print("bs", bs)
print("opt", opt)
print("sh", sh)
print("sa", sa)
print("sym", sym)
print("beta", beta)
print("act_fn", act_fn)
print("fp16", fp16)
print("pool", pool)
print("runs", runs)
print("model", model)<jupyter_output>lr 0.008
size 128
sqrmom 0.99
mom 0.9
eps 1e-06
epochs 15
bs 64
opt ranger
sh 0.0
sa 0
sym 0
beta 0.0
act_fn <class 'fastai2.layers.Mish'>
fp16 0
pool <function MaxPool at 0x7f87caaece60>
runs 3
model <function xresnet34 at 0x7f87ca8fa8c0><jupyter_text>Get Items From Folder So before we do anything, let's create some helper methods that will give us only the training sets that we would like.<jupyter_code>source = untar_data(URLs.IMAGEWANG_160)
# transforms are the same for each experiment
item_tfms=[RandomResizedCrop(size, min_scale=0.35), FlipItem(0.5), RandomCutout]
batch_tfms=RandomErasing(p=0.9, max_count=3, sh=sh) if sh else None
def get_all_items(path):
    return get_files(path, extensions='.JPEG', recurse=True)

def get_train_items(path):
    return get_files(path/'train', extensions='.JPEG', recurse=True)

def get_unsup_items(path):
    return get_files(path/'unsup', extensions='.JPEG', recurse=True)

def get_valid_items(path):
    return get_files(path/'val', extensions='.JPEG', recurse=True)

def get_train_and_unsup(path):
    return get_train_items(path) + get_unsup_items(path)

def get_train_items_that_are_present_in_val(path):
    """
    We first get a list of all classes in /val
    Then we use that list to get all the examples of each class from /train
    """
    val = source/'val'
    validation_classes = [path.name for path in val.iterdir()]
    
    train_files = L()
    for class_name in validation_classes:
        items = get_files(path/'train'/class_name, extensions='.JPEG', recurse=True)
        train_files = train_files + items
        
    return train_files

all_items = get_all_items(untar_data(URLs.IMAGEWANG_160))
train_items = get_train_items(untar_data(URLs.IMAGEWANG_160))
unsup_items = get_unsup_items(untar_data(URLs.IMAGEWANG_160))
valid_items = get_valid_items(untar_data(URLs.IMAGEWANG_160))

print("All Files:\t{}".format(len(all_items)))
print("Train Files:\t{}".format(len(train_items)))
print("Unsup Files:\t{}".format(len(unsup_items)))
print("Valid Files:\t{}".format(len(valid_items)))
print()

train_and_unsup_items = get_train_and_unsup(untar_data(URLs.IMAGEWANG_160))
print("Train+Unsup Files: {}".format(len(train_and_unsup_items)))
train_in_valid_items = get_train_items_that_are_present_in_val(untar_data(URLs.IMAGEWANG_160))
print("Train(in validation set)+Unsup Files: {}".format(len(train_in_valid_items)))<jupyter_output>All Files:	26348
Train Files:	14669
Unsup Files:	7750
Valid Files:	3929

Train+Unsup Files: 22419
Train(in validation set)+Unsup Files: 1275<jupyter_text>Train with all data in `/train`, `/unsup` and `/val`<jupyter_code>dblock = DataBlock(blocks=(ImageBlock(cls=PILImageInput), ImageBlock),
                   splitter=RandomSplitter(valid_pct=0),
                   get_items=get_all_items, 
                   get_y=lambda o: o,
                   item_tfms=item_tfms,
                   batch_tfms=batch_tfms)

dbunch =  dblock.dataloaders(source, path=source, bs=bs)

#CHANGE: We're predicting pixel values, so we're just going to predict an output for each RGB channel
dbunch.vocab = ['R', 'G', 'B']

print("Training Size:", len(dbunch.train_ds))
print("Validation Size:", len(dbunch.valid_ds))
for run in range(runs):
    learn = unet_learner(dbunch, model, pretrained=False, opt_func=opt_func, metrics=[], loss_func=MSELoss())

    if fp16: learn = learn.to_fp16()
    cbs = []
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)

    #Save model backbone
    torch.save(learn.model[0].state_dict(), 'all_train_unsup_val_pretext_{}.pth'.format(run))
    
    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output><empty_output><jupyter_text>Train with all data in `/train` and `/unsup`<jupyter_code>dblock = DataBlock(blocks=(ImageBlock(cls=PILImageInput), ImageBlock),
                   splitter=RandomSplitter(valid_pct=0),
                   get_items=get_train_and_unsup, 
                   get_y=lambda o: o,
                   item_tfms=item_tfms, 
                   batch_tfms=batch_tfms)

dbunch =  dblock.dataloaders(source, path=source, bs=bs)

#CHANGE: We're predicting pixel values, so we're just going to predict an output for each RGB channel
dbunch.vocab = ['R', 'G', 'B']

print("Training Size:", len(dbunch.train_ds))
print("Validation Size:", len(dbunch.valid_ds))
for run in range(runs):
    learn = unet_learner(dbunch, model, pretrained=False, opt_func=opt_func, metrics=[], loss_func=MSELoss())

    if fp16: learn = learn.to_fp16()
    cbs = []
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)

    #Save model backbone
    torch.save(learn.model[0].state_dict(), 'all_train_unsup_pretext_{}.pth'.format(run))
    
    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output><empty_output><jupyter_text>Train with all data in `/train`<jupyter_code>dblock = DataBlock(blocks=(ImageBlock(cls=PILImageInput), ImageBlock),
                   splitter=RandomSplitter(valid_pct=0),
                   get_items=get_train_items, 
                   get_y=lambda o: o,
                   item_tfms=item_tfms, 
                   batch_tfms=batch_tfms)

dbunch =  dblock.dataloaders(source, path=source, bs=bs)

#CHANGE: We're predicting pixel values, so we're just going to predict an output for each RGB channel
dbunch.vocab = ['R', 'G', 'B']

print("Training Size:", len(dbunch.train_ds))
print("Validation Size:", len(dbunch.valid_ds))
for run in range(runs):
    learn = unet_learner(dbunch, model, pretrained=False, opt_func=opt_func, metrics=[], loss_func=MSELoss())

    if fp16: learn = learn.to_fp16()
    cbs = []
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)

    #Save model backbone
    torch.save(learn.model[0].state_dict(), 'all_train_pretext_{}.pth'.format(run))
    
    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output><empty_output><jupyter_text>Train with partial data from `/train`<jupyter_code>dblock = DataBlock(blocks=(ImageBlock(cls=PILImageInput), ImageBlock),
                   splitter=RandomSplitter(valid_pct=0),
                   get_items=get_train_items_that_are_present_in_val, 
                   get_y=lambda o: o,
                   item_tfms=item_tfms, 
                   batch_tfms=batch_tfms)

dbunch =  dblock.dataloaders(source, path=source, bs=bs)

#CHANGE: We're predicting pixel values, so we're just going to predict an output for each RGB channel
dbunch.vocab = ['R', 'G', 'B']

print("Training Size:", len(dbunch.train_ds))
print("Validation Size:", len(dbunch.valid_ds))
for run in range(runs):
    learn = unet_learner(dbunch, model, pretrained=False, opt_func=opt_func, metrics=[], loss_func=MSELoss())

    if fp16: learn = learn.to_fp16()
    cbs = []
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)

    #Save model backbone
    torch.save(learn.model[0].state_dict(), 'partial_train_pretext_{}.pth'.format(run))
    
    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output><empty_output><jupyter_text>Downstream Task: Image网 Now that we've trained models on our pretext tasks, let's compare the performance of each model against one another.<jupyter_code>def get_dbunch(size, bs, sh=0., workers=None):
    if size<=224: 
        path = URLs.IMAGEWANG_160
    else: 
        path = URLs.IMAGEWANG
    source = untar_data(path)
    if workers is None: workers = min(8, num_cpus())
    item_tfms=[RandomResizedCrop(size, min_scale=0.35), FlipItem(0.5)]
    batch_tfms=RandomErasing(p=0.9, max_count=3, sh=sh) if sh else None
    
    dblock = DataBlock(blocks=(ImageBlock, CategoryBlock),
                       splitter=GrandparentSplitter(valid_name='val'),
                       get_items=get_image_files, get_y=parent_label,
                       item_tfms=item_tfms, batch_tfms=batch_tfms)
    
    return dblock.dataloaders(source, path=source, bs=bs, num_workers=workers)
dbunch = get_dbunch(size, bs)<jupyter_output><empty_output><jupyter_text>Random Baseline<jupyter_code>for run in range(runs):
    print(f'Run: {run}')
    learn = Learner(dbunch, model(c_out=20, pretrained=False, act_cls=torch.nn.ReLU, sa=sa, sym=sym, pool=pool), opt_func=opt_func, \
            metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy())

    if fp16: learn = learn.to_fp16()
    cbs = []
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)
        
    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output>Run: 0<jupyter_text>Results:- Run 1: 0.523797- Run 2: 0.514889- Run 3: 0.529651Average: **52.3%**  All data in `/train`, `/unsup` and `/val`<jupyter_code>for run in range(runs):
    print(f'Run: {run}')
    learn = Learner(dbunch, model(c_out=20, pretrained=False, act_cls=torch.nn.ReLU, sa=sa, sym=sym, pool=pool), opt_func=opt_func, \
            metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy())

    if fp16: learn = learn.to_fp16()
    cbs = []

    # Load weights generated from training on our pretext task
    model_path = 'all_train_unsup_val_pretext_' + str(run) + '.pth'
    state_dict = torch.load(model_path)
    # HACK: If we don't have all of the parameters for our learner, we get an error
    linear_layer = learn.model[-1]
    state_dict['11.weight'] = linear_layer.weight
    state_dict['11.bias'] = linear_layer.bias

    learn.model.load_state_dict(state_dict)

    learn.freeze()
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)

    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output>Run: 0<jupyter_text>Results:- Run 1: 0.567065- Run 2: 0.567320- Run 3: 0.555612Average: **56.3%**  All data in `/train` and `/unsup`<jupyter_code>for run in range(runs):
    print(f'Run: {run}')
    learn = Learner(dbunch, model(c_out=20, pretrained=False, act_cls=torch.nn.ReLU, sa=sa, sym=sym, pool=pool), opt_func=opt_func, \
            metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy())

    if fp16: learn = learn.to_fp16()
    cbs = []

    # Load weights generated from training on our pretext task
    model_path = 'all_train_unsup_pretext_' + str(run) + '.pth'
    state_dict = torch.load(model_path)
    # HACK: If we don't have all of the parameters for our learner, we get an error
    linear_layer = learn.model[-1]
    state_dict['11.weight'] = linear_layer.weight
    state_dict['11.bias'] = linear_layer.bias

    learn.model.load_state_dict(state_dict)

    learn.freeze()
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)
    
    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output>Run: 0<jupyter_text>Results:- Run 1: 0.564011- Run 2: 0.556121- Run 3: 0.564520Average: **56.2%**  All data in `/train`<jupyter_code>for run in range(runs):
    print(f'Run: {run}')
    learn = Learner(dbunch, model(c_out=20, pretrained=False, act_cls=torch.nn.ReLU, sa=sa, sym=sym, pool=pool), opt_func=opt_func, \
            metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy())

    if fp16: learn = learn.to_fp16()
    cbs = []

    # Load weights generated from training on our pretext task
    model_path = 'all_train_pretext_' + str(run) + '.pth'
    state_dict = torch.load(model_path)
    # HACK: If we don't have all of the parameters for our learner, we get an error
    linear_layer = learn.model[-1]
    state_dict['11.weight'] = linear_layer.weight
    state_dict['11.bias'] = linear_layer.bias

    learn.model.load_state_dict(state_dict)

    learn.freeze()
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)

    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output>Run: 0<jupyter_text>Results:- Run 1: 0.557139- Run 2: 0.554594- Run 3: 0.557903Average: **55.7%**  Partial data from `/train`<jupyter_code>for run in range(runs):
    print(f'Run: {run}')
    learn = Learner(dbunch, model(c_out=20, pretrained=False, act_cls=torch.nn.ReLU, sa=sa, sym=sym, pool=pool), opt_func=opt_func, \
            metrics=[accuracy,top_k_accuracy], loss_func=LabelSmoothingCrossEntropy())

    if fp16: learn = learn.to_fp16()
    cbs = []

    # Load weights generated from training on our pretext task
    model_path = 'partial_train_pretext_' + str(run) + '.pth'
    state_dict = torch.load(model_path)
    # HACK: If we don't have all of the parameters for our learner, we get an error
    linear_layer = learn.model[-1]
    state_dict['11.weight'] = linear_layer.weight
    state_dict['11.bias'] = linear_layer.bias

    learn.model.load_state_dict(state_dict)

    learn.freeze()
    learn.fit_flat_cos(epochs, lr, wd=1e-2, cbs=cbs)

    del learn
    torch.cuda.empty_cache() 
    gc.collect()<jupyter_output>Run: 0<jupyter_text>Results:- Run 1: 0.531942- Run 2: 0.515398- Run 3: 0.544413Average: **53.1%**  Results: - Random: **52.3%**- Partial `/train`: **53.1%**- All `/train`: **55.7%**- All `/train` and `/unsup` : **56.2%**- All `/train`,`/unsup` and `/val` : **56.3%**<jupyter_code># No pretraining, train subset, all train, train + unsup, train + unsup + val
x = [0, 1275, 14669, 22419, 26348]
y = [0.523,  0.531, 0.557, 0.562, 0.563]
plt.title("Size of pretext task vs. downstream performance")
plt.xlabel("# of images in pretext task")
plt.ylabel("Downstream accuray")
plt.scatter(x,y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13230869849574886, 'fraction_numerical': 0.0289077828646174, 'mean_word_length': 4.21877133105802, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '419796', 'license': 'MIT', 'n_tokens_mistral': 5944, 'n_tokens_neox': 5552, 'n_words': 1254, 'path': '02_InpaintingVaryDatasetSize/01_InpaintingWithVariedDatasetSize.ipynb', 'repo_name': 'JoshVarty/SelfSupervisedLearning', 'size': '15290'}