<jupyter_start><jupyter_text>Interacting with CLIPThis is a self-contained notebook that shows how to download and run CLIP models, calculate the similarity between arbitrary image and text inputs, and perform zero-shot image classifications.  Preparation for ColabMake sure you're running a GPU runtime; if not, select "GPU" as the hardware accelerator in Runtime > Change Runtime Type in the menu. The next cells will print the CUDA version of the runtime if it has a GPU, and install PyTorch 1.7.1.<jupyter_code>import subprocess

CUDA_version = [s for s in subprocess.check_output(["nvcc", "--version"]).decode("UTF-8").split(", ") if s.startswith("release")][0].split(" ")[-1]
print("CUDA version:", CUDA_version)

if CUDA_version == "10.0":
    torch_version_suffix = "+cu100"
elif CUDA_version == "10.1":
    torch_version_suffix = "+cu101"
elif CUDA_version == "10.2":
    torch_version_suffix = ""
else:
    torch_version_suffix = "+cu110"
! pip install torch==1.7.1{torch_version_suffix} torchvision==0.8.2{torch_version_suffix} -f https://download.pytorch.org/whl/torch_stable.html ftfy regex
import numpy as np
import torch

print("Torch version:", torch.__version__)<jupyter_output>Torch version: 1.7.1+cu110<jupyter_text>Downloading the modelCLIP models are distributed as TorchScript modules.<jupyter_code>MODELS = {
    "ViT-B/32":       "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
}
! wget {MODELS["ViT-B/32"]} -O model.pt
model = torch.jit.load("model.pt").cuda().eval()
input_resolution = model.input_resolution.item()
context_length = model.context_length.item()
vocab_size = model.vocab_size.item()

print("Model parameters:", f"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}")
print("Input resolution:", input_resolution)
print("Context length:", context_length)
print("Vocab size:", vocab_size)<jupyter_output>Model parameters: 151,277,313
Input resolution: 224
Context length: 77
Vocab size: 49408<jupyter_text>Image PreprocessingWe resize the input images and center-crop them to conform with the image resolution that the model expects. Before doing so, we will normalize the pixel intensity using the dataset mean and standard deviation.<jupyter_code>from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
from PIL import Image

preprocess = Compose([
    Resize(input_resolution, interpolation=Image.BICUBIC),
    CenterCrop(input_resolution),
    ToTensor()
])

image_mean = torch.tensor([0.48145466, 0.4578275, 0.40821073]).cuda()
image_std = torch.tensor([0.26862954, 0.26130258, 0.27577711]).cuda()<jupyter_output><empty_output><jupyter_text>Text PreprocessingWe use a case-insensitive tokenizer. The tokenizer code is hidden in the second cell below<jupyter_code>! pip install ftfy regex
! wget https://openaipublic.azureedge.net/clip/bpe_simple_vocab_16e6.txt.gz -O bpe_simple_vocab_16e6.txt.gz
#@title

import gzip
import html
import os
from functools import lru_cache

import ftfy
import regex as re


@lru_cache()
def bytes_to_unicode():
    """
    Returns list of utf-8 byte and a corresponding list of unicode strings.
    The reversible bpe codes work on unicode strings.
    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
    This is a signficant percentage of your normal, say, 32K bpe vocab.
    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
    And avoids mapping to whitespace/control characters the bpe code barfs on.
    """
    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
    cs = bs[:]
    n = 0
    for b in range(2**8):
        if b not in bs:
            bs.append(b)
            cs.append(2**8+n)
            n += 1
    cs = [chr(n) for n in cs]
    return dict(zip(bs, cs))


def get_pairs(word):
    """Return set of symbol pairs in a word.
    Word is represented as tuple of symbols (symbols being variable-length strings).
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs


def basic_clean(text):
    text = ftfy.fix_text(text)
    text = html.unescape(html.unescape(text))
    return text.strip()


def whitespace_clean(text):
    text = re.sub(r'\s+', ' ', text)
    text = text.strip()
    return text


class SimpleTokenizer(object):
    def __init__(self, bpe_path: str = "bpe_simple_vocab_16e6.txt.gz"):
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
        merges = merges[1:49152-256-2+1]
        merges = [tuple(merge.split()) for merge in merges]
        vocab = list(bytes_to_unicode().values())
        vocab = vocab + [v+'</w>' for v in vocab]
        for merge in merges:
            vocab.append(''.join(merge))
        vocab.extend(['<|startoftext|>', '<|endoftext|>'])
        self.encoder = dict(zip(vocab, range(len(vocab))))
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}
        self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)

    def bpe(self, token):
        if token in self.cache:
            return self.cache[token]
        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
        pairs = get_pairs(word)

        if not pairs:
            return token+'</w>'

        while True:
            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                    new_word.extend(word[i:j])
                    i = j
                except:
                    new_word.extend(word[i:])
                    break

                if word[i] == first and i < len(word)-1 and word[i+1] == second:
                    new_word.append(first+second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = ' '.join(word)
        self.cache[token] = word
        return word

    def encode(self, text):
        bpe_tokens = []
        text = whitespace_clean(basic_clean(text)).lower()
        for token in re.findall(self.pat, text):
            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
        return bpe_tokens

    def decode(self, tokens):
        text = ''.join([self.decoder[token] for token in tokens])
        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
        return text<jupyter_output><empty_output><jupyter_text>Zero-shot classification for Oxford buildings dataset.  Getting dataset<jupyter_code>%mkdir data
%cd data
%mkdir buildings
# the following line should be modified if you run the notebook on your computer
# change directory to data where you will store the dataset
%cd /content/data/buildings
!wget https://www.robots.ox.ac.uk/~vgg/data/oxbuildings/oxbuild_images.tgz
!tar -zxvf oxbuild_images.tgz
# create directory if it does not exist
def check_dir(dir_path):
    dir_path = dir_path.replace('//','/')
    os.makedirs(dir_path, exist_ok=True)
import os
from os import listdir
from os.path import isfile, join

data_dir = '/content/data'

import shutil
for f in listdir(join(data_dir, 'buildings')):
    # create directory if necessary
    path = join(data_dir, f[:-11])
    check_dir(path)
    # copy to target directory
    src_path = join(data_dir, 'buildings', f)
    targ_path = join(path, f)
    shutil.move(src_path, targ_path)
try:
    shutil.move('/content/data/oxbuild', '/content/sample_data')
    print('The directory has been moved successfully')
except:
    print('The directory has already been moved')
from pathlib import Path
try:
    shutil.move('/content/data/buildings', '/content/sample_data')
    print('The directory has been moved successfully')
except:
    print('The directory has already been moved')
!pwd
from torchvision import datasets
data_dir = '/content/data'
dsets = datasets.ImageFolder(data_dir, preprocess)
# with preprocessing
# dsets = datasets.ImageFolder(data_dir, preprocess)
%ls /content/data
data_loader = torch.utils.data.DataLoader(dsets, batch_size=64, shuffle=True, num_workers=6)
from_text = True
text_descriptions = {}
if from_text:
  s = ""
  with open("/content/oxford2.txt") as fp:
      Lines = fp.readlines()
      for line in Lines:
          if line.startswith('---'):
              if s != "":
                  text_descriptions.update({building: s})
                  print("Line {}: {}".format(building, s))
              s = ""
              building = line[3:-4]
          else:
              s += ' ' + line[:-2] 
  text_descriptions.update({building: s})
else:
  classes = [s.replace('_', ' ') for s in dsets.classes]
  text_descriptions = {label: f"This is a photo of {label}" for label in classes}
print(*dsets.classes, sep='\n')
text_descriptions
# text_descriptions['All Souls College']
!pwd
%cd /content/
tokenizer = SimpleTokenizer()

sot_token = tokenizer.encoder['<|startoftext|>']
eot_token = tokenizer.encoder['<|endoftext|>']

text_tokens = [[sot_token] + tokenizer.encode(desc)[:65] + [eot_token] 
               for name, desc in text_descriptions.items()]
text_input = torch.zeros(len(text_tokens), model.context_length, dtype=torch.long)

for i, tokens in enumerate(text_tokens):
    text_input[i, :len(tokens)] = torch.tensor(tokens)

text_input = text_input.cuda()
text_input.shape
with torch.no_grad():
    text_features = model.encode_text(text_input).float()
    text_features /= text_features.norm(dim=-1, keepdim=True)
import matplotlib.pyplot as plt

def imshow(inp, title=None):
#   Imshow for Tensor.
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = np.clip(std * inp + mean, 0,1)
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
acc, acc_top5, count = 0, 0, 0
for i, (x, label) in enumerate(data_loader):
    
    with torch.no_grad():
        image_input = torch.tensor(np.stack(x)).cuda()
        # image_input -= image_mean[:, None, None]
        # image_input /= image_std[:, None, None]
        image_features = model.encode_image(image_input).float()

    text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
    top_probs, top_labels = text_probs.cpu().topk(5, dim=-1)

    acc += (top_labels[:, 0] == label).sum()
    # print(top_labels[:, 0], label)
    count += len(label)
    for k in range(5):
        acc_top5 += (top_labels[:,k] == label).sum()

    if i % 10 == 0: 
      print('%3d' %i, 'batches processed |', 
            'Accuracy is %.3f |' %(acc / count), 
            'Top 5 accuracy is %.3f' %(acc_top5 / count))
print('-'*85)
print('Total accuracy is %.3f |' %(acc / count), 
      'Top 5 accuracy is %.3f' %(acc_top5 / count))
print('Accuracy is %.3f |' %(acc / count), 
      'Top 5 accuracy is %.3f' %(acc_top5 / count))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.136169141706519, 'fraction_numerical': 0.02365970299521772, 'mean_word_length': 3.15186346220829, 'pattern_counts': {'":': 5, '<': 35, '<?xml version=': 0, '>': 34, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '21944', 'license': 'MIT', 'n_tokens_mistral': 4169, 'n_tokens_neox': 3942, 'n_words': 1028, 'path': 'CLIP_with_Oxford_buildings_dataset_v1.ipynb', 'repo_name': 'IemProg/Project-CLIP', 'size': '11919'}