






import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD


def 				_a     (   )       ->      List[str]:
						"""simple docstring"""







						lowerCAmelCase__       					=  """https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"""
						lowerCAmelCase__       					=  Image.open(requests.get(_UpperCAmelCase			,      stream=_UpperCAmelCase      ).raw      ).convert("RGB"      )

						return image


def 				_a     (   UpperCamelCase_    : Union[str, Any]      )       ->      Optional[Any]:
						"""simple docstring"""







						lowerCAmelCase__       					=  []
						# fmt: off

						# vision encoder
						rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding")      )
						rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding")      )
						rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight")      )
						rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias")      )
						rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight")      )
						rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias")      )

						for i in range(config.vision_config.num_hidden_layers      ):
												rename_keys.append((F"visual_encoder.blocks.{i}.norm1.weight", F"vision_model.encoder.layers.{i}.layer_norm1.weight")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.norm1.bias", F"vision_model.encoder.layers.{i}.layer_norm1.bias")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.norm2.weight", F"vision_model.encoder.layers.{i}.layer_norm2.weight")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.norm2.bias", F"vision_model.encoder.layers.{i}.layer_norm2.bias")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.attn.qkv.weight", F"vision_model.encoder.layers.{i}.self_attn.qkv.weight")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.weight", F"vision_model.encoder.layers.{i}.self_attn.projection.weight",)      )
												rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.bias", F"vision_model.encoder.layers.{i}.self_attn.projection.bias")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.weight", F"vision_model.encoder.layers.{i}.mlp.fc1.weight")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.bias", F"vision_model.encoder.layers.{i}.mlp.fc1.bias")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.weight", F"vision_model.encoder.layers.{i}.mlp.fc2.weight")      )
												rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.bias", F"vision_model.encoder.layers.{i}.mlp.fc2.bias")      )

						# QFormer
						rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.embeddings.layernorm.weight")      )
						rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.embeddings.layernorm.bias")      )

						# fmt: on
						return rename_keys


def 				_a     (   UpperCamelCase_    : Any			,      UpperCamelCase_    : Union[str, Any]			,      UpperCamelCase_    : int      )       ->      str:
						"""simple docstring"""







						lowerCAmelCase__       					=  dct.pop(_UpperCAmelCase      )
						lowerCAmelCase__       					=  val


def 				_a     (   UpperCamelCase_    : Optional[int]			,      UpperCamelCase_    : Any      )       ->      Optional[int]:
						"""simple docstring"""







						for i in range(config.vision_config.num_hidden_layers      ):
												# read in original q and v biases
												lowerCAmelCase__       					=  state_dict.pop(F"visual_encoder.blocks.{i}.attn.q_bias"      )
												lowerCAmelCase__       					=  state_dict.pop(F"visual_encoder.blocks.{i}.attn.v_bias"      )

												# next, set bias in the state dict
												lowerCAmelCase__       					=  torch.cat((q_bias, torch.zeros_like(_UpperCAmelCase			,      requires_grad=_UpperCAmelCase      ), v_bias)      )
												lowerCAmelCase__       					=  qkv_bias


def 				_a     (   UpperCamelCase_    : Dict      )       ->      List[Any]:
						"""simple docstring"""







						lowerCAmelCase__       					=  364 if """coco""" in model_name else 224
						lowerCAmelCase__       					=  InstructBlipVisionConfig(image_size=_UpperCAmelCase      ).to_dict()

						# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
						# seems like flan-T5 models don't have bos_token_id properly set?
						if "t5-xl" in model_name:
												lowerCAmelCase__       					=  TaConfig.from_pretrained("google/flan-t5-xl"			,      dense_act_fn="gelu"			,      bos_token_id=1      ).to_dict()
						elif "t5-xxl" in model_name:
												lowerCAmelCase__       					=  TaConfig.from_pretrained("google/flan-t5-xxl"			,      dense_act_fn="gelu"			,      bos_token_id=1      ).to_dict()
						elif "vicuna-7b" in model_name:
												lowerCAmelCase__       					=  LlamaConfig.from_pretrained("decapoda-research/llama-7b-hf"			,      vocab_size=32_001      ).to_dict()
						elif "vicuna-13b" in model_name:
												lowerCAmelCase__       					=  LlamaConfig.from_pretrained("decapoda-research/llama-13b-hf"			,      vocab_size=32_001      ).to_dict()
						else:
												raise ValueError("Model name not supported"      )

						# the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
						lowerCAmelCase__       					=  InstructBlipQFormerConfig(vocab_size=30_523      ).to_dict()
						lowerCAmelCase__       					=  InstructBlipConfig(vision_config=_UpperCAmelCase			,      text_config=_UpperCAmelCase			,      qformer_config=_UpperCAmelCase      )

						return config, image_size


@torch.no_grad()
def 				_a     (   UpperCamelCase_    : Union[str, Any]			,      UpperCamelCase_    : Any=None			,      UpperCamelCase_    : List[str]=False      )       ->      Dict:
						"""simple docstring"""







						lowerCAmelCase__       					=  AutoTokenizer.from_pretrained("bert-base-uncased"			,      truncation_side="left"      )
						qformer_tokenizer.add_special_tokens({"bos_token": "[DEC]"}      )

						if "t5" in model_name:
												lowerCAmelCase__       					=  TaTokenizerFast.from_pretrained("google/flan-t5-xl"			,      truncation_side="left"      )
						elif "vicuna" in model_name:
												# the following was used in the original implementation:
												# tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
												# tokenizer.add_special_tokens({"pad_token": "[PAD]"})
												# tokenizer.add_special_tokens({"bos_token": "</s>"})
												# tokenizer.add_special_tokens({"eos_token": "</s>"})
												# tokenizer.add_special_tokens({"unk_token": "</s>"})
												lowerCAmelCase__       					=  LlamaTokenizerFast.from_pretrained(
												    "huggyllama/llama-7b"			,      truncation_side="left"			,      bos_token="</s>"			,      unk_token="</s>"      )
												tokenizer.add_special_tokens({"pad_token": "[PAD]"}      )

						lowerCAmelCase__       					=  get_blipa_config(_UpperCAmelCase      )
						lowerCAmelCase__       					=  InstructBlipForConditionalGeneration(_UpperCAmelCase      ).eval()

						lowerCAmelCase__       					=  {
						    """instructblip-vicuna-7b""": ("""blip2_vicuna_instruct""", """vicuna7b"""),
						    """instructblip-vicuna-13b""": ("""blip2_vicuna_instruct""", """vicuna13b"""),
						    """instructblip-flan-t5-xl""": ("""blip2_t5_instruct""", """flant5xl"""),
						    """instructblip-flan-t5-xxl""": ("""blip2_t5_instruct""", """flant5xxl"""),
						}

						lowerCAmelCase__       					=  model_name_to_original[model_name]

						# load original model
						print("Loading original model..."      )
						lowerCAmelCase__       					=  """cuda:1""" if torch.cuda.is_available() else """cpu"""
						lowerCAmelCase__       					=  """cuda:2""" if torch.cuda.is_available() else """cpu"""
						lowerCAmelCase__       					=  load_model_and_preprocess(
						    name=_UpperCAmelCase			,      model_type=_UpperCAmelCase			,      is_eval=_UpperCAmelCase			,      device=_UpperCAmelCase      )
						original_model.eval()
						print("Done!"      )

						# update state dict keys
						lowerCAmelCase__       					=  original_model.state_dict()
						lowerCAmelCase__       					=  create_rename_keys(_UpperCAmelCase      )
						for src, dest in rename_keys:
												rename_key(_UpperCAmelCase			,      _UpperCAmelCase			,      _UpperCAmelCase      )

						# some keys can be renamed efficiently
						for key, val in state_dict.copy().items():
												lowerCAmelCase__       					=  state_dict.pop(_UpperCAmelCase      )
												if key.startswith("Qformer.bert"      ):
																		lowerCAmelCase__       					=  key.replace("Qformer.bert"			,      "qformer"      )
												if "attention.self" in key:
																		lowerCAmelCase__       					=  key.replace("self"			,      "attention"      )
												if "llm_proj" in key:
																		lowerCAmelCase__       					=  key.replace("llm_proj"			,      "language_projection"      )
												if "t5_proj" in key:
																		lowerCAmelCase__       					=  key.replace("t5_proj"			,      "language_projection"      )
												if key.startswith("llm_model"      ):
																		lowerCAmelCase__       					=  key.replace("llm_model"			,      "language_model"      )
												if key.startswith("t5"      ):
																		lowerCAmelCase__       					=  key.replace("t5"			,      "language"      )
												lowerCAmelCase__       					=  val

						# read in qv biases
						read_in_q_v_bias(_UpperCAmelCase			,      _UpperCAmelCase      )

						# note: weights get loaded in torch.float32 by default
						hf_model.load_state_dict(_UpperCAmelCase			,      strict=_UpperCAmelCase      )

						lowerCAmelCase__       					=  load_demo_image()
						lowerCAmelCase__       					=  """What is unusual about this image?"""

						# create processor
						lowerCAmelCase__       					=  BlipImageProcessor(
						    size={"height": image_size, "width": image_size}			,      image_mean=_UpperCAmelCase			,      image_std=_UpperCAmelCase      )
						lowerCAmelCase__       					=  InstructBlipProcessor(
						    image_processor=_UpperCAmelCase			,      tokenizer=_UpperCAmelCase			,      qformer_tokenizer=_UpperCAmelCase			,      )
						lowerCAmelCase__       					=  processor(images=_UpperCAmelCase			,      text=_UpperCAmelCase			,      return_tensors="pt"      ).to(_UpperCAmelCase      )

						# make sure processor creates exact same pixel values
						lowerCAmelCase__       					=  vis_processors["""eval"""](_UpperCAmelCase      ).unsqueeze(0      ).to(_UpperCAmelCase      )
						lowerCAmelCase__       					=  inputs.pixel_values
						assert torch.allclose(original_pixel_values.to(pixel_values.device      )			,      _UpperCAmelCase      )

						original_model.to(_UpperCAmelCase      )
						hf_model.to(_UpperCAmelCase      )
						with torch.no_grad():
												if "vicuna" in model_name:
																		lowerCAmelCase__       					=  original_model({"image": original_pixel_values, "text_input": [prompt]}      ).logits
																		lowerCAmelCase__       					=  hf_model(**_UpperCAmelCase      ).logits
												else:
																		lowerCAmelCase__       					=  original_model(
																		    {"image": original_pixel_values, "text_input": [prompt], "text_output": ["\n"]}      ).logits
																		lowerCAmelCase__       					=  tokenizer("\n"			,      return_tensors="pt"      ).input_ids.to(_UpperCAmelCase      )
																		lowerCAmelCase__       					=  label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id			,      -100      )
																		lowerCAmelCase__       					=  hf_model(**_UpperCAmelCase			,      labels=_UpperCAmelCase      ).logits

						print("First values of original logits:"			,      original_logits[0, :3, :3]      )
						print("First values of HF logits:"			,      logits[0, :3, :3]      )

						# assert values
						assert original_logits.shape == logits.shape
						lowerCAmelCase__       					=  1e-4 if """vicuna""" in model_name else 1e-5
						assert torch.allclose(original_logits.to(logits.device      )			,      _UpperCAmelCase			,      atol=_UpperCAmelCase      )
						print("Looks ok!"      )

						print("Generating with original model..."      )
						lowerCAmelCase__       					=  original_model.generate({"image": original_pixel_values, "prompt": prompt}			,      num_beams=5      )

						# important: we need to cast the weights of the HF model to the appropriate type
						print("Generating with HF model..."      )
						lowerCAmelCase__       					=  hf_model.generate(
						    **_UpperCAmelCase			,      do_sample=_UpperCAmelCase			,      num_beams=5			,      max_length=256			,      min_length=1			,      top_p=0.9			,      repetition_penalty=1.5			,      length_penalty=1.0			,      temperature=1			,      )
						if "vicuna" in model_name:
												# convert output id 0 to 2 (eos_token_id)
												# TODO add this in the generate method?
												lowerCAmelCase__       					=  2
						print("Original generation:"			,      _UpperCAmelCase      )
						lowerCAmelCase__       					=  processor.batch_decode(_UpperCAmelCase			,      skip_special_tokens=_UpperCAmelCase      )
						lowerCAmelCase__       					=  [text.strip() for text in output_text]
						print("HF generation:"			,      _UpperCAmelCase      )

						if pytorch_dump_folder_path is not None:
												processor.save_pretrained(_UpperCAmelCase      )
												hf_model.save_pretrained(_UpperCAmelCase      )

						if push_to_hub:
												processor.push_to_hub(F"Salesforce/{model_name}"      )
												hf_model.push_to_hub(F"Salesforce/{model_name}"      )


if __name__ == "__main__":
	a_							=			argparse.ArgumentParser()
	a_							=			[
	    '''instructblip-vicuna-7b''',
	    '''instructblip-vicuna-13b''',
	    '''instructblip-flan-t5-xl''',
	    '''instructblip-flan-t5-xxl''',
	]
	parser.add_argument(
	    '''--model_name''',
	    default='''instructblip-flan-t5-xl''',
	    choices=choices,
	    type=str,
	    help='''Path to hf config.json of model to convert''',
	)
	parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
	parser.add_argument(
	    '''--push_to_hub''',
	    action='''store_true''',
	    help='''Whether to push the model and processor to the hub after converting''',
	)

	a_							=			parser.parse_args()

	convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)










'''simple docstring'''







A__:    Optional[int]					    =       [4, 1, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
A__:    Any					    =       [3, 7, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
A__:    int					    =       {
    0: '''Sunday''',
    1: '''Monday''',
    2: '''Tuesday''',
    3: '''Wednesday''',
    4: '''Thursday''',
    5: '''Friday''',
    6: '''Saturday''',
}


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	int    ,_UpperCAmelCase	:	int    ,_UpperCAmelCase	:	int  )  ->			str:
       assert len(str(_UpperCAmelCase  )  ) > 2, "year should be in YYYY format"
       assert 1 <= month <= 12, "month should be between 1 to 12"
       assert 1 <= day <= 31, "day should be between 1 to 31"

       # Doomsday algorithm:
       _a    : List[str]     					=year // 100
       _a    : List[str]     					=(5 * (century % 4) + 2) % 7
       _a    : Optional[int]     					=year % 100
       _a    : Any     					=centurian % 12
       _a    : int     					=(
           (centurian // 12) + centurian_m + (centurian_m // 4) + century_anchor
       ) % 7
       _a    : Optional[Any]     					=(
           DOOMSDAY_NOT_LEAP[month - 1]
           if (year % 4 != 0) or (centurian == 0 and (year % 400) == 0)
           else DOOMSDAY_LEAP[month - 1]
       )
       _a    : str     					=(dooms_day + day - day_anchor) % 7
       return WEEK_DAY_NAMES[week_day]


if __name__ == "__main__":
   import doctest

   doctest.testmod()


