


"""simple docstring"""






import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
def 					A_      (      ):






     """simple docstring"""

     _a					= '''https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png'''
     _a					= Image.open(requests.get(A_,   stream=A_				).raw				).convert('''RGB'''				)

     return image
def 					A_      (      _lowerCAmelCase   :    List[str]				):






     """simple docstring"""

     _a					= []
     # fmt: off

     # vision encoder
     rename_keys.append(('''visual_encoder.cls_token''', '''vision_model.embeddings.class_embedding''')				)
     rename_keys.append(('''visual_encoder.pos_embed''', '''vision_model.embeddings.position_embedding''')				)
     rename_keys.append(('''visual_encoder.patch_embed.proj.weight''', '''vision_model.embeddings.patch_embedding.weight''')				)
     rename_keys.append(('''visual_encoder.patch_embed.proj.bias''', '''vision_model.embeddings.patch_embedding.bias''')				)
     rename_keys.append(('''ln_vision.weight''', '''vision_model.post_layernorm.weight''')				)
     rename_keys.append(('''ln_vision.bias''', '''vision_model.post_layernorm.bias''')				)

     for i in range(config.vision_config.num_hidden_layers				):
          rename_keys.append((f'visual_encoder.blocks.{i}.norm1.weight', f'vision_model.encoder.layers.{i}.layer_norm1.weight')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.norm1.bias', f'vision_model.encoder.layers.{i}.layer_norm1.bias')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.norm2.weight', f'vision_model.encoder.layers.{i}.layer_norm2.weight')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.norm2.bias', f'vision_model.encoder.layers.{i}.layer_norm2.bias')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.attn.qkv.weight', f'vision_model.encoder.layers.{i}.self_attn.qkv.weight')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.attn.proj.weight', f'vision_model.encoder.layers.{i}.self_attn.projection.weight',)				)
          rename_keys.append((f'visual_encoder.blocks.{i}.attn.proj.bias', f'vision_model.encoder.layers.{i}.self_attn.projection.bias')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc1.weight', f'vision_model.encoder.layers.{i}.mlp.fc1.weight')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc1.bias', f'vision_model.encoder.layers.{i}.mlp.fc1.bias')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc2.weight', f'vision_model.encoder.layers.{i}.mlp.fc2.weight')				)
          rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc2.bias', f'vision_model.encoder.layers.{i}.mlp.fc2.bias')				)

     # QFormer
     rename_keys.append(('''Qformer.bert.embeddings.LayerNorm.weight''', '''qformer.layernorm.weight''')				)
     rename_keys.append(('''Qformer.bert.embeddings.LayerNorm.bias''', '''qformer.layernorm.bias''')				)

     # fmt: on
     return rename_keys
def 					A_      (      _lowerCAmelCase   :    List[str],   _lowerCAmelCase   :    Any,   _lowerCAmelCase   :    List[Any]				):






     """simple docstring"""

     _a					= dct.pop(A_				)
     _a					= val
def 					A_      (      _lowerCAmelCase   :    Tuple,   _lowerCAmelCase   :    Any				):






     """simple docstring"""

     for i in range(config.vision_config.num_hidden_layers				):
          # read in original q and v biases
          _a					= state_dict.pop(f'visual_encoder.blocks.{i}.attn.q_bias'				)
          _a					= state_dict.pop(f'visual_encoder.blocks.{i}.attn.v_bias'				)

          # next, set bias in the state dict
          _a					= torch.cat((q_bias, torch.zeros_like(A_,   requires_grad=A_				), v_bias)				)
          _a					= qkv_bias
def 					A_      (      _lowerCAmelCase   :    Optional[Any],   _lowerCAmelCase   :    Optional[Any]				):






     """simple docstring"""

     _a					= 3_64 if '''coco''' in model_name else 2_24
     _a					= BlipaVisionConfig(image_size=A_				).to_dict()

     # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
     # seems like flan-T5 models don't have bos_token_id properly set?
     if "opt-2.7b" in model_name:
          _a					= OPTConfig.from_pretrained('''facebook/opt-2.7b''',   eos_token_id=A_				).to_dict()
     elif "opt-6.7b" in model_name:
          _a					= OPTConfig.from_pretrained('''facebook/opt-6.7b''',   eos_token_id=A_				).to_dict()
     elif "t5-xl" in model_name:
          _a					= TaConfig.from_pretrained('''google/flan-t5-xl''',   dense_act_fn='''gelu''',   bos_token_id=1				).to_dict()
     elif "t5-xxl" in model_name:
          _a					= TaConfig.from_pretrained('''google/flan-t5-xxl''',   dense_act_fn='''gelu''',   bos_token_id=1				).to_dict()

     _a					= BlipaConfig(vision_config=A_,   text_config=A_				)

     return config, image_size
@torch.no_grad()
def 					A_      (      _lowerCAmelCase   :    Optional[int],   _lowerCAmelCase   :    List[str]=None,   _lowerCAmelCase   :    List[str]=False				):






     """simple docstring"""

     _a					= (
         AutoTokenizer.from_pretrained('''facebook/opt-2.7b'''				)
         if '''opt''' in model_name
         else AutoTokenizer.from_pretrained('''google/flan-t5-xl'''				)
     )
     _a					= tokenizer('''\n''',   add_special_tokens=A_				).input_ids[0]
     _a					= get_blipa_config(A_,   eos_token_id=A_				)

     _a					= BlipaForConditionalGeneration(A_				).eval()

     _a					= {
         '''blip2-opt-2.7b''': ('''blip2_opt''', '''pretrain_opt2.7b'''),
         '''blip2-opt-6.7b''': ('''blip2_opt''', '''pretrain_opt6.7b'''),
         '''blip2-opt-2.7b-coco''': ('''blip2_opt''', '''caption_coco_opt2.7b'''),
         '''blip2-opt-6.7b-coco''': ('''blip2_opt''', '''caption_coco_opt6.7b'''),
         '''blip2-flan-t5-xl''': ('''blip2_t5''', '''pretrain_flant5xl'''),
         '''blip2-flan-t5-xl-coco''': ('''blip2_t5''', '''caption_coco_flant5xl'''),
         '''blip2-flan-t5-xxl''': ('''blip2_t5''', '''pretrain_flant5xxl'''),
     }

     _a					= model_name_to_original[model_name]

     # load original model
     print('''Loading original model...'''				)
     _a					= '''cuda''' if torch.cuda.is_available() else '''cpu'''
     _a					= load_model_and_preprocess(
         name=A_,   model_type=A_,   is_eval=A_,   device=A_				)
     original_model.eval()
     print('''Done!'''				)

     # update state dict keys
     _a					= original_model.state_dict()
     _a					= create_rename_keys(A_				)
     for src, dest in rename_keys:
          rename_key(A_,   A_,   A_				)

     # some keys can be renamed efficiently
     for key, val in state_dict.copy().items():
          _a					= state_dict.pop(A_				)
          if key.startswith('''Qformer.bert'''				):
               _a					= key.replace('''Qformer.bert''',   '''qformer'''				)
          if "attention.self" in key:
               _a					= key.replace('''self''',   '''attention'''				)
          if "opt_proj" in key:
               _a					= key.replace('''opt_proj''',   '''language_projection'''				)
          if "t5_proj" in key:
               _a					= key.replace('''t5_proj''',   '''language_projection'''				)
          if key.startswith('''opt'''				):
               _a					= key.replace('''opt''',   '''language'''				)
          if key.startswith('''t5'''				):
               _a					= key.replace('''t5''',   '''language'''				)
          _a					= val

     # read in qv biases
     read_in_q_v_bias(A_,   A_				)

     _a					= hf_model.load_state_dict(A_,   strict=A_				)
     assert len(A_				) == 0
     assert unexpected_keys == ["qformer.embeddings.position_ids"]

     _a					= load_demo_image()
     _a					= vis_processors['''eval'''](A_				).unsqueeze(0				).to(A_				)
     _a					= tokenizer(['''\n'''],   return_tensors='''pt'''				).input_ids.to(A_				)

     # create processor
     _a					= BlipImageProcessor(
         size={'''height''': image_size, '''width''': image_size},   image_mean=A_,   image_std=A_				)
     _a					= BlipaProcessor(image_processor=A_,   tokenizer=A_				)
     _a					= processor(images=A_,   return_tensors='''pt'''				).pixel_values.to(A_				)

     # make sure processor creates exact same pixel values
     assert torch.allclose(A_,   A_				)

     original_model.to(A_				)
     hf_model.to(A_				)
     with torch.no_grad():
          if "opt" in model_name:
               _a					= original_model({'''image''': original_pixel_values, '''text_input''': ['''''']}				).logits
               _a					= hf_model(A_,   A_				).logits
          else:
               _a					= original_model(
                   {'''image''': original_pixel_values, '''text_input''': ['''\n'''], '''text_output''': ['''\n''']}				).logits
               _a					= input_ids.masked_fill(input_ids == tokenizer.pad_token_id,   -1_00				)
               _a					= hf_model(A_,   A_,   labels=A_				).logits

     assert original_logits.shape == logits.shape
     print('''First values of original logits:''',   original_logits[0, :3, :3]				)
     print('''First values of HF logits:''',   logits[0, :3, :3]				)

     # assert values
     if model_name == "blip2-flan-t5-xl":
          _a					= torch.tensor(
              [[-4_1.5_8_5_0, -4.4_4_4_0, -8.9_9_2_2], [-4_7.4_3_2_2, -5.9_1_4_3, -1.7_3_4_0]],   device=A_				)
          assert torch.allclose(logits[0, :3, :3],   A_,   atol=1e-4				)
     elif model_name == "blip2-flan-t5-xl-coco":
          _a					= torch.tensor(
              [[-5_7.0_1_0_9, -9.8_9_6_7, -1_2.6_2_8_0], [-6_8.6_5_7_8, -1_2.7_1_9_1, -1_0.5_0_6_5]],   device=A_				)
     else:
          # cast to same type
          _a					= logits.dtype
          assert torch.allclose(original_logits.to(A_				),   A_,   atol=1e-2				)
     print('''Looks ok!'''				)

     print('''Generating a caption...'''				)
     _a					= ''''''
     _a					= tokenizer(A_,   return_tensors='''pt'''				).input_ids.to(A_				)

     _a					= original_model.generate({'''image''': original_pixel_values}				)
     _a					= hf_model.generate(
         A_,   A_,   do_sample=A_,   num_beams=5,   max_length=30,   min_length=1,   top_p=0.9,   repetition_penalty=1.0,   length_penalty=1.0,   temperature=1,   )
     print('''Original generation:''',   A_				)
     _a					= input_ids.shape[1]
     _a					= processor.batch_decode(outputs[:, prompt_length:],   skip_special_tokens=A_				)
     _a					= [text.strip() for text in output_text]
     print('''HF generation:''',   A_				)

     if pytorch_dump_folder_path is not None:
          processor.save_pretrained(A_				)
          hf_model.save_pretrained(A_				)

     if push_to_hub:
          processor.push_to_hub(f'nielsr/{model_name}'				)
          hf_model.push_to_hub(f'nielsr/{model_name}'				)


if __name__ == "__main__":
   __snake_case				=				argparse.ArgumentParser()
   __snake_case				=				[
       '''blip2-opt-2.7b''',
       '''blip2-opt-6.7b''',
       '''blip2-opt-2.7b-coco''',
       '''blip2-opt-6.7b-coco''',
       '''blip2-flan-t5-xl''',
       '''blip2-flan-t5-xl-coco''',
       '''blip2-flan-t5-xxl''',
   ]
   parser.add_argument(
       '''--model_name''',
       default='''blip2-opt-2.7b''',
       choices=choices,
       type=str,
       help='''Path to hf config.json of model to convert''',
   )
   parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
   parser.add_argument(
       '''--push_to_hub''',
       action='''store_true''',
       help='''Whether to push the model and processor to the hub after converting''',
   )

   __snake_case				=				parser.parse_args()

   convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)


"""simple docstring"""





def   __SCREAMING_SNAKE_CASE  (      A_						,     A_ ):
	return numa ^ numa < 0


if __name__ == "__main__":
							import doctest

							doctest.testmod()




