


'''simple docstring'''







import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD


def 	SCREAMING_SNAKE_CASE_					(  )  ->			List[str]:
       _a    : Optional[int]     					="""https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"""
       _a    : Union[str, Any]     					=Image.open(requests.get(_UpperCAmelCase    ,stream=_UpperCAmelCase  ).raw  ).convert("""RGB"""  )

       return image


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Union[str, Any]  )  ->			Optional[Any]:
       _a    : List[str]     					=[]
       # fmt: off

       # vision encoder
       rename_keys.append(("""visual_encoder.cls_token""", """vision_model.embeddings.class_embedding""")  )
       rename_keys.append(("""visual_encoder.pos_embed""", """vision_model.embeddings.position_embedding""")  )
       rename_keys.append(("""visual_encoder.patch_embed.proj.weight""", """vision_model.embeddings.patch_embedding.weight""")  )
       rename_keys.append(("""visual_encoder.patch_embed.proj.bias""", """vision_model.embeddings.patch_embedding.bias""")  )
       rename_keys.append(("""ln_vision.weight""", """vision_model.post_layernorm.weight""")  )
       rename_keys.append(("""ln_vision.bias""", """vision_model.post_layernorm.bias""")  )

       for i in range(config.vision_config.num_hidden_layers  ):
              rename_keys.append((F"visual_encoder.blocks.{i}.norm1.weight", F"vision_model.encoder.layers.{i}.layer_norm1.weight")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.norm1.bias", F"vision_model.encoder.layers.{i}.layer_norm1.bias")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.norm2.weight", F"vision_model.encoder.layers.{i}.layer_norm2.weight")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.norm2.bias", F"vision_model.encoder.layers.{i}.layer_norm2.bias")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.attn.qkv.weight", F"vision_model.encoder.layers.{i}.self_attn.qkv.weight")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.weight", F"vision_model.encoder.layers.{i}.self_attn.projection.weight",)  )
              rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.bias", F"vision_model.encoder.layers.{i}.self_attn.projection.bias")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.weight", F"vision_model.encoder.layers.{i}.mlp.fc1.weight")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.bias", F"vision_model.encoder.layers.{i}.mlp.fc1.bias")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.weight", F"vision_model.encoder.layers.{i}.mlp.fc2.weight")  )
              rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.bias", F"vision_model.encoder.layers.{i}.mlp.fc2.bias")  )

       # QFormer
       rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.weight""", """qformer.embeddings.layernorm.weight""")  )
       rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.bias""", """qformer.embeddings.layernorm.bias""")  )

       # fmt: on
       return rename_keys


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Any    ,_UpperCAmelCase	:	Union[str, Any]    ,_UpperCAmelCase	:	int  )  ->			str:
       _a    : Optional[int]     					=dct.pop(_UpperCAmelCase  )
       _a    : Tuple     					=val


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[int]    ,_UpperCAmelCase	:	Any  )  ->			Optional[int]:
       for i in range(config.vision_config.num_hidden_layers  ):
              # read in original q and v biases
              _a    : Optional[int]     					=state_dict.pop(F"visual_encoder.blocks.{i}.attn.q_bias"  )
              _a    : Any     					=state_dict.pop(F"visual_encoder.blocks.{i}.attn.v_bias"  )

              # next, set bias in the state dict
              _a    : List[str]     					=torch.cat((q_bias, torch.zeros_like(_UpperCAmelCase    ,requires_grad=_UpperCAmelCase  ), v_bias)  )
              _a    : Union[str, Any]     					=qkv_bias


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Dict  )  ->			List[Any]:
       _a    : List[Any]     					=364 if """coco""" in model_name else 224
       _a    : Tuple     					=InstructBlipVisionConfig(image_size=_UpperCAmelCase  ).to_dict()

       # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
       # seems like flan-T5 models don't have bos_token_id properly set?
       if "t5-xl" in model_name:
              _a    : List[str]     					=TaConfig.from_pretrained("""google/flan-t5-xl"""    ,dense_act_fn="""gelu"""    ,bos_token_id=1  ).to_dict()
       elif "t5-xxl" in model_name:
              _a    : Optional[Any]     					=TaConfig.from_pretrained("""google/flan-t5-xxl"""    ,dense_act_fn="""gelu"""    ,bos_token_id=1  ).to_dict()
       elif "vicuna-7b" in model_name:
              _a    : Union[str, Any]     					=LlamaConfig.from_pretrained("""decapoda-research/llama-7b-hf"""    ,vocab_size=32001  ).to_dict()
       elif "vicuna-13b" in model_name:
              _a    : Optional[int]     					=LlamaConfig.from_pretrained("""decapoda-research/llama-13b-hf"""    ,vocab_size=32001  ).to_dict()
       else:
              raise ValueError("""Model name not supported"""  )

       # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
       _a    : int     					=InstructBlipQFormerConfig(vocab_size=30523  ).to_dict()
       _a    : Tuple     					=InstructBlipConfig(vision_config=_UpperCAmelCase    ,text_config=_UpperCAmelCase    ,qformer_config=_UpperCAmelCase  )

       return config, image_size


@torch.no_grad()
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Union[str, Any]    ,_UpperCAmelCase	:	Any=None    ,_UpperCAmelCase	:	List[str]=False  )  ->			Dict:
       _a    : Optional[Any]     					=AutoTokenizer.from_pretrained("""bert-base-uncased"""    ,truncation_side="""left"""  )
       qformer_tokenizer.add_special_tokens({"""bos_token""": """[DEC]"""}  )

       if "t5" in model_name:
              _a    : Union[str, Any]     					=TaTokenizerFast.from_pretrained("""google/flan-t5-xl"""    ,truncation_side="""left"""  )
       elif "vicuna" in model_name:
              # the following was used in the original implementation:
              # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
              # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
              # tokenizer.add_special_tokens({"bos_token": "</s>"})
              # tokenizer.add_special_tokens({"eos_token": "</s>"})
              # tokenizer.add_special_tokens({"unk_token": "</s>"})
              _a    : str     					=LlamaTokenizerFast.from_pretrained(
                  """huggyllama/llama-7b"""    ,truncation_side="""left"""    ,bos_token="""</s>"""    ,unk_token="""</s>"""  )
              tokenizer.add_special_tokens({"""pad_token""": """[PAD]"""}  )

       _a					,    _a    : Tuple     					=get_blipa_config(_UpperCAmelCase  )
       _a    : Tuple     					=InstructBlipForConditionalGeneration(_UpperCAmelCase  ).eval()

       _a    : List[str]     					={
           """instructblip-vicuna-7b""": ("""blip2_vicuna_instruct""", """vicuna7b"""),
           """instructblip-vicuna-13b""": ("""blip2_vicuna_instruct""", """vicuna13b"""),
           """instructblip-flan-t5-xl""": ("""blip2_t5_instruct""", """flant5xl"""),
           """instructblip-flan-t5-xxl""": ("""blip2_t5_instruct""", """flant5xxl"""),
       }

       _a					,    _a    : Optional[Any]     					=model_name_to_original[model_name]

       # load original model
       print("""Loading original model..."""  )
       _a    : Optional[Any]     					="""cuda:1""" if torch.cuda.is_available() else """cpu"""
       _a    : str     					="""cuda:2""" if torch.cuda.is_available() else """cpu"""
       _a					,    _a					,    _a    : int     					=load_model_and_preprocess(
           name=_UpperCAmelCase    ,model_type=_UpperCAmelCase    ,is_eval=_UpperCAmelCase    ,device=_UpperCAmelCase  )
       original_model.eval()
       print("""Done!"""  )

       # update state dict keys
       _a    : Tuple     					=original_model.state_dict()
       _a    : Dict     					=create_rename_keys(_UpperCAmelCase  )
       for src, dest in rename_keys:
              rename_key(_UpperCAmelCase    ,_UpperCAmelCase    ,_UpperCAmelCase  )

       # some keys can be renamed efficiently
       for key, val in state_dict.copy().items():
              _a    : List[str]     					=state_dict.pop(_UpperCAmelCase  )
              if key.startswith("""Qformer.bert"""  ):
                     _a    : int     					=key.replace("""Qformer.bert"""    ,"""qformer"""  )
              if "attention.self" in key:
                     _a    : str     					=key.replace("""self"""    ,"""attention"""  )
              if "llm_proj" in key:
                     _a    : List[str]     					=key.replace("""llm_proj"""    ,"""language_projection"""  )
              if "t5_proj" in key:
                     _a    : Union[str, Any]     					=key.replace("""t5_proj"""    ,"""language_projection"""  )
              if key.startswith("""llm_model"""  ):
                     _a    : Dict     					=key.replace("""llm_model"""    ,"""language_model"""  )
              if key.startswith("""t5"""  ):
                     _a    : Dict     					=key.replace("""t5"""    ,"""language"""  )
              _a    : Union[str, Any]     					=val

       # read in qv biases
       read_in_q_v_bias(_UpperCAmelCase    ,_UpperCAmelCase  )

       # note: weights get loaded in torch.float32 by default
       hf_model.load_state_dict(_UpperCAmelCase    ,strict=_UpperCAmelCase  )

       _a    : Optional[int]     					=load_demo_image()
       _a    : Tuple     					="""What is unusual about this image?"""

       # create processor
       _a    : List[Any]     					=BlipImageProcessor(
           size={"""height""": image_size, """width""": image_size}    ,image_mean=_UpperCAmelCase    ,image_std=_UpperCAmelCase  )
       _a    : List[str]     					=InstructBlipProcessor(
           image_processor=_UpperCAmelCase    ,tokenizer=_UpperCAmelCase    ,qformer_tokenizer=_UpperCAmelCase    ,)
       _a    : Tuple     					=processor(images=_UpperCAmelCase    ,text=_UpperCAmelCase    ,return_tensors="""pt"""  ).to(_UpperCAmelCase  )

       # make sure processor creates exact same pixel values
       _a    : Tuple     					=vis_processors["""eval"""](_UpperCAmelCase  ).unsqueeze(0  ).to(_UpperCAmelCase  )
       _a    : str     					=inputs.pixel_values
       assert torch.allclose(original_pixel_values.to(pixel_values.device  )    ,_UpperCAmelCase  )

       original_model.to(_UpperCAmelCase  )
       hf_model.to(_UpperCAmelCase  )
       with torch.no_grad():
              if "vicuna" in model_name:
                     _a    : List[Any]     					=original_model({"""image""": original_pixel_values, """text_input""": [prompt]}  ).logits
                     _a    : str     					=hf_model(**_UpperCAmelCase  ).logits
              else:
                     _a    : str     					=original_model(
                         {"""image""": original_pixel_values, """text_input""": [prompt], """text_output""": ["""\n"""]}  ).logits
                     _a    : Optional[int]     					=tokenizer("""\n"""    ,return_tensors="""pt"""  ).input_ids.to(_UpperCAmelCase  )
                     _a    : List[str]     					=label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id    ,-100  )
                     _a    : List[Any]     					=hf_model(**_UpperCAmelCase    ,labels=_UpperCAmelCase  ).logits

       print("""First values of original logits:"""    ,original_logits[0, :3, :3]  )
       print("""First values of HF logits:"""    ,logits[0, :3, :3]  )

       # assert values
       assert original_logits.shape == logits.shape
       _a    : str     					=1e-4 if """vicuna""" in model_name else 1e-5
       assert torch.allclose(original_logits.to(logits.device  )    ,_UpperCAmelCase    ,atol=_UpperCAmelCase  )
       print("""Looks ok!"""  )

       print("""Generating with original model..."""  )
       _a    : Union[str, Any]     					=original_model.generate({"""image""": original_pixel_values, """prompt""": prompt}    ,num_beams=5  )

       # important: we need to cast the weights of the HF model to the appropriate type
       print("""Generating with HF model..."""  )
       _a    : Dict     					=hf_model.generate(
           **_UpperCAmelCase    ,do_sample=_UpperCAmelCase    ,num_beams=5    ,max_length=256    ,min_length=1    ,top_p=0.9    ,repetition_penalty=1.5    ,length_penalty=1.0    ,temperature=1    ,)
       if "vicuna" in model_name:
              # convert output id 0 to 2 (eos_token_id)
              # TODO add this in the generate method?
              _a    : Any     					=2
       print("""Original generation:"""    ,_UpperCAmelCase  )
       _a    : Optional[int]     					=processor.batch_decode(_UpperCAmelCase    ,skip_special_tokens=_UpperCAmelCase  )
       _a    : int     					=[text.strip() for text in output_text]
       print("""HF generation:"""    ,_UpperCAmelCase  )

       if pytorch_dump_folder_path is not None:
              processor.save_pretrained(_UpperCAmelCase  )
              hf_model.save_pretrained(_UpperCAmelCase  )

       if push_to_hub:
              processor.push_to_hub(F"Salesforce/{model_name}"  )
              hf_model.push_to_hub(F"Salesforce/{model_name}"  )


if __name__ == "__main__":
   A__:    Optional[Any]					    =       argparse.ArgumentParser()
   A__:    Any					    =       [
       '''instructblip-vicuna-7b''',
       '''instructblip-vicuna-13b''',
       '''instructblip-flan-t5-xl''',
       '''instructblip-flan-t5-xxl''',
   ]
   parser.add_argument(
       '''--model_name''',
       default='''instructblip-flan-t5-xl''',
       choices=choices,
       type=str,
       help='''Path to hf config.json of model to convert''',
   )
   parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
   parser.add_argument(
       '''--push_to_hub''',
       action='''store_true''',
       help='''Whether to push the model and processor to the hub after converting''',
   )

   A__:    int					    =       parser.parse_args()

   convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)






'''simple docstring'''







class A__       :






      def __init__( self			:List[Any]	)       ->					None:




             '''simple docstring'''
             _a    : dict[str, TrieNode]     					={}  # Mapping from char to TrieNode
             _a    : List[str]     					=False






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:list[str]	)       ->					None:




             '''simple docstring'''
             for word in words:
                    self.insert(SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:str	)       ->					None:




             '''simple docstring'''
             _a    : str     					=self
             for char in word:
                    if char not in curr.nodes:
                           _a    : Dict     					=TrieNode()
                    _a    : List[Any]     					=curr.nodes[char]
             _a    : int     					=True






      def __UpperCAmelCase       ( self			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:str	)       ->					bool:




             '''simple docstring'''
             _a    : int     					=self
             for char in word:
                    if char not in curr.nodes:
                           return False
                    _a    : List[Any]     					=curr.nodes[char]
             return curr.is_leaf






      def __UpperCAmelCase       ( self			:Dict     ,						SCREAMING_SNAKE_CASE			:str	)       ->					None:




             '''simple docstring'''

             def _delete(SCREAMING_SNAKE_CASE			:TrieNode     ,						SCREAMING_SNAKE_CASE			:str     ,						SCREAMING_SNAKE_CASE			:int	) -> bool:
                    if index == len(SCREAMING_SNAKE_CASE	):
                           # If word does not exist
                           if not curr.is_leaf:
                                  return False
                           _a    : Any     					=False
                           return len(curr.nodes	) == 0
                    _a    : int     					=word[index]
                    _a    : int     					=curr.nodes.get(SCREAMING_SNAKE_CASE	)
                    # If char not in current trie node
                    if not char_node:
                           return False
                    # Flag to check if node can be deleted
                    _a    : List[Any]     					=_delete(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE     ,						index + 1	)
                    if delete_curr:
                           del curr.nodes[char]
                           return len(curr.nodes	) == 0
                    return delete_curr

             _delete(self     ,						SCREAMING_SNAKE_CASE     ,						0	)


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	TrieNode    ,_UpperCAmelCase	:	str  )  ->			None:
       if node.is_leaf:
              print(_UpperCAmelCase    ,end=""" """  )

       for key, value in node.nodes.items():
              print_words(_UpperCAmelCase    ,word + key  )


def 	SCREAMING_SNAKE_CASE_					(  )  ->			bool:
       _a    : List[str]     					="""banana bananas bandana band apple all beast""".split()
       _a    : List[Any]     					=TrieNode()
       root.insert_many(_UpperCAmelCase  )
       # print_words(root, "")
       assert all(root.find(_UpperCAmelCase  ) for word in words  )
       assert root.find("""banana"""  )
       assert not root.find("""bandanas"""  )
       assert not root.find("""apps"""  )
       assert root.find("""apple"""  )
       assert root.find("""all"""  )
       root.delete("""all"""  )
       assert not root.find("""all"""  )
       root.delete("""banana"""  )
       assert not root.find("""banana"""  )
       assert root.find("""bananas"""  )
       return True


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	str    ,_UpperCAmelCase	:	bool  )  ->			None:
       print(str(_UpperCAmelCase  )    ,"""works!""" if passes else """doesn't work :("""  )


def 	SCREAMING_SNAKE_CASE_					(  )  ->			None:
       assert test_trie()


def 	SCREAMING_SNAKE_CASE_					(  )  ->			None:
       print_results("""Testing trie functionality"""    ,test_trie()  )


if __name__ == "__main__":
   main()


