<jupyter_start><jupyter_text>Set-up environment<jupyter_code>!pip install -q transformers
!git clone https://github.com/microsoft/unilm.git
!pip install -q einops
!pip install -q DALL-E
%cd unilm/beit<jupyter_output>/content/unilm/beit<jupyter_text>Read imageLet's say we have the following image.<jupyter_code>from PIL import Image
import requests

url = 'http://images.cocodataset.org/val2017/000000039769.jpg'
image = Image.open(requests.get(url, stream=True).raw)

image<jupyter_output><empty_output><jupyter_text>TransformsAs seen [here](https://github.com/microsoft/unilm/blob/00a147f4ce71c0b21589c07ef8ffca406c12e5be/beit/engine_for_pretraining.pyL44), during pre-training, each batch consists of 3 things:- Original image (resized + normalized): `pixel_values`- Image prepared for DALL-E encoder (map_pixels): `pixel_values_dall_e`- Mask (indicating which visual tokens are masked): `bool_masked_pos`<jupyter_code>from transformers import BeitFeatureExtractor

feature_extractor = BeitFeatureExtractor()

# create input 1 (pixel_values)
pixel_values = feature_extractor(image, return_tensors="pt").pixel_values
pixel_values.shape
from torchvision import transforms
from dall_e.utils import map_pixels

# step 2: visual tokens transform => in order to get input_ids
visual_token_transform = transforms.Compose([
                transforms.Resize((112,112), transforms.InterpolationMode.LANCZOS),
                transforms.ToTensor(),
                map_pixels,
            ])
pixel_values_dall_e = visual_token_transform(image).unsqueeze(0)
pixel_values_dall_e.shape
# Download the tokenizer weight from OpenAI's DALL-E
!mkdir -p dall_e_tokenizer
!wget -o dall_e_tokenizer/encoder.pkl https://cdn.openai.com/dall-e/encoder.pkl
from transformers import BeitForMaskedImageModeling

model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k")
from masking_generator import MaskingGenerator

window_size = model.beit.embeddings.patch_embeddings.patch_shape
num_masking_patches = 75
max_mask_patches_per_block = None
min_mask_patches_per_block = 16

# generating mask for the corresponding image
mask_generator = MaskingGenerator(
            window_size, num_masking_patches=num_masking_patches,
            max_num_patches=max_mask_patches_per_block,
            min_num_patches=min_mask_patches_per_block,
        )
import torch

# create input 2: bool_masked_pos
bool_masked_pos = mask_generator()
bool_masked_pos = torch.from_numpy(bool_masked_pos).unsqueeze(0)
bool_masked_pos.shape
from dall_e import map_pixels, load_model 
import torch

# step 2: get input_ids from OpenAI's DALL-E
device = torch.device('cpu')
encoder = load_model("https://cdn.openai.com/dall-e/encoder.pkl", device)
pixel_values_dall_e.shape
with torch.no_grad():
    z_logits = encoder(pixel_values_dall_e)
    input_ids = torch.argmax(z_logits, axis=1).flatten(1)
    bool_masked_pos = bool_masked_pos.flatten(1).to(torch.bool)
    labels = input_ids[bool_masked_pos]
input_ids.shape
labels.shape<jupyter_output><empty_output><jupyter_text>Forward pass<jupyter_code>outputs = model(pixel_values, bool_masked_pos)
labels
predictions = outputs.logits[bool_masked_pos].argmax(-1)
predictions<jupyter_output><empty_output><jupyter_text>Integration test<jupyter_code>from transformers import BeitForMaskedImageModeling
from PIL import Image
from torchvision import transforms

url = 'http://images.cocodataset.org/val2017/000000039769.jpg'
image = Image.open(requests.get(url, stream=True).raw)

# prepare for model (simply resize + normalize)
mean = (0.5, 0.5, 0.5)
std = (0.5, 0.5, 0.5)
transform = transforms.Compose([transforms.Resize((224, 224)), 
                                transforms.ToTensor(), 
                                transforms.Normalize(mean, std)])
pixel_values = transform(image).unsqueeze(0)
pixel_values.shape
pixel_values[0,:3,:3,:3]
import torch

# prepare bool_masked_pos
bool_masked_pos = torch.ones((1, 196), dtype=torch.bool)

# forward pass
outputs = model(pixel_values, bool_masked_pos)
outputs.logits.shape
outputs.logits[bool_masked_pos][:3,:3]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09772065955383123, 'fraction_numerical': 0.027643064985451018, 'mean_word_length': 5.568471337579618, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '317064', 'license': 'MIT', 'n_tokens_mistral': 1501, 'n_tokens_neox': 1411, 'n_words': 257, 'path': 'BEiT/Understanding_BeitForMaskedImageModeling.ipynb', 'repo_name': 'FrancescoSaverioZuppichini/Transformers-Tutorials', 'size': '4124'}