<jupyter_start><jupyter_text>CLIP COCO-Captions Tutorial This notebook demonstrates how to finetune CLIP ViT-B/32 model on a single GPU.<jupyter_code>from fastai.vision.all import *
from fastai.distributed import *
from self_supervised.multimodal.clip import *
import clip
# from zero_optimizer import ZeroRedundancyOptimizer # only works with multi-gpu / distributed training<jupyter_output><empty_output><jupyter_text>Download [COCO](https://cocodataset.org/download) 2014 Train, Valid and Annotations<jupyter_code>trainpath = Path("../../../coco/train2014/")
validpath = Path("../../../coco/val2014/")
annospath = Path("../../../coco/annotations/") 
train_images = get_image_files(trainpath)
valid_images = get_image_files(validpath)
len(train_images), len(valid_images)
caption_paths = annospath.ls().filter(lambda o: 'captions' in o.name)
fn2captions = {}
for p in caption_paths:
    caps = json.loads(open(p).read())
    id2fn = {o['id']: o['file_name'] for o in caps['images']}
    fn2cap = {id2fn[o['image_id']]: o['caption'] for o in caps['annotations']}
    fn2captions.update(fn2cap)

assert len(fn2captions) == (len(train_images) + len(valid_images))
all_images = train_images[:10000] + valid_images[:5000]; len(all_images),len(fn2captions)
def read_image(fn): return PILImage.create(fn)
def read_text(fn): return fn2captions[fn.name]
def dummy_targ(o): return 0 
clip_stats = ([0.48145466, 0.4578275, 0.40821073], [0.26862954, 0.26130258, 0.27577711])
clip_tokenizer = ClipTokenizer()
size,bs = 224,128

split_func = lambda fn: True if "val2014" in str(fn) else False
dsets = Datasets(all_images, tfms=[read_image, read_text, dummy_targ], 
                 n_inp=2, 
                 splits=FuncSplitter(split_func)(all_images))
item_tfms = [RandomResizedCrop(size, min_scale=0.9), clip_tokenizer, ToTensor()]

batch_tfms = [IntToFloatTensor, Normalize.from_stats(*clip_stats)]
train_dl = TfmdDL(dsets.train, shuffle=True, bs=bs, after_item=item_tfms, after_batch=batch_tfms, drop_last=True)
valid_dl = TfmdDL(dsets.valid, shuffle=False, bs=bs*2, after_item=item_tfms, after_batch=batch_tfms)
dls = DataLoaders(train_dl, valid_dl, device=default_device())
for i in range(5): dls.train_ds[i][0].show(title=dls.train_ds[i][1])
clip_trainer_cb = CLIPTrainer()
cbs = [clip_trainer_cb]
opt_func = ranger

arch = 'vitb32'
do_finetune = True
use_grad_check = True
grad_check_nchunks = 2
finetune_modelname = 'ViT-B/32'

vitb32_config_dict = vitb32_config(size, clip_tokenizer.context_length, clip_tokenizer.vocab_size)
clip_model = CLIP(**vitb32_config_dict, checkpoint=use_grad_check, checkpoint_nchunks=grad_check_nchunks)
if do_finetune:
    print("Loading pretrained model..")
    clip_pretrained_model, _ = clip.load(finetune_modelname, jit=False)
    clip_model.load_state_dict(clip_pretrained_model.state_dict())
learner = Learner(dls, clip_model, loss_func=noop, cbs=cbs, opt_func=opt_func,
              metrics=[RetrievalAtK(k=5), 
                       RetrievalAtK(k=20), 
                       RetrievalAtK(k="mean"),
                       RetrievalAtK(k="median")])
learner.to_fp16();
learner.fit_flat_cos(10, 1e-5, pct_start=0.25)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13253768844221106, 'fraction_numerical': 0.03957286432160804, 'mean_word_length': 5.526639344262295, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '266171', 'license': 'Apache-2.0', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1177, 'n_words': 206, 'path': 'examples/multimodal/01 - training_clip_coco.ipynb', 'repo_name': 'jimmiemunyi/self_supervised', 'size': '3184'}