






"""simple docstring"""
import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD

def 				snake_case      (   ):
			UpperCAmelCase_     : List[Any]			     =					"https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png"
			UpperCAmelCase_     : Dict			     =					Image.open(requests.get(A__	,stream=A__						).raw						).convert("RGB"						)

			return image

def 				snake_case      (   A__						):
			UpperCAmelCase_     : Dict			     =					[]
			# fmt: off

			# vision encoder
			rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding")						)
			rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding")						)
			rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight")						)
			rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias")						)
			rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight")						)
			rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias")						)

			for i in range(config.vision_config.num_hidden_layers						):
						rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.weight""", F"""vision_model.encoder.layers.{i}.layer_norm1.weight""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.bias""", F"""vision_model.encoder.layers.{i}.layer_norm1.bias""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.weight""", F"""vision_model.encoder.layers.{i}.layer_norm2.weight""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.bias""", F"""vision_model.encoder.layers.{i}.layer_norm2.bias""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.attn.qkv.weight""", F"""vision_model.encoder.layers.{i}.self_attn.qkv.weight""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.weight""", F"""vision_model.encoder.layers.{i}.self_attn.projection.weight""",)						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.bias""", F"""vision_model.encoder.layers.{i}.self_attn.projection.bias""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc1.weight""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc1.bias""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc2.weight""")						)
						rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc2.bias""")						)

			# QFormer
			rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.layernorm.weight")						)
			rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.layernorm.bias")						)

			# fmt: on
			return rename_keys

def 				snake_case      (   A__	,A__	,A__						):
			UpperCAmelCase_     : Optional[Any]			     =					dct.pop(A__						)
			UpperCAmelCase_     : Dict			     =					val

def 				snake_case      (   A__	,A__						):
			for i in range(config.vision_config.num_hidden_layers						):
						# read in original q and v biases
						UpperCAmelCase_     : Union[str, Any]			     =					state_dict.pop(F"""visual_encoder.blocks.{i}.attn.q_bias"""						)
						UpperCAmelCase_     : Tuple			     =					state_dict.pop(F"""visual_encoder.blocks.{i}.attn.v_bias"""						)

						# next, set bias in the state dict
						UpperCAmelCase_     : Union[str, Any]			     =					torch.cat((q_bias, torch.zeros_like(A__	,requires_grad=A__						), v_bias)						)
						UpperCAmelCase_     : Any			     =					qkv_bias

def 				snake_case      (   A__	,A__						):
			UpperCAmelCase_     : int			     =					3_64 if "coco" in model_name else 2_24
			UpperCAmelCase_     : List[str]			     =					BlipaVisionConfig(image_size=A__						).to_dict()

			# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
			# seems like flan-T5 models don't have bos_token_id properly set?
			if "opt-2.7b" in model_name:
						UpperCAmelCase_     : Any			     =					OPTConfig.from_pretrained("facebook/opt-2.7b"	,eos_token_id=A__						).to_dict()
			elif "opt-6.7b" in model_name:
						UpperCAmelCase_     : Any			     =					OPTConfig.from_pretrained("facebook/opt-6.7b"	,eos_token_id=A__						).to_dict()
			elif "t5-xl" in model_name:
						UpperCAmelCase_     : List[Any]			     =					TaConfig.from_pretrained("google/flan-t5-xl"	,dense_act_fn="gelu"	,bos_token_id=1						).to_dict()
			elif "t5-xxl" in model_name:
						UpperCAmelCase_     : List[str]			     =					TaConfig.from_pretrained("google/flan-t5-xxl"	,dense_act_fn="gelu"	,bos_token_id=1						).to_dict()

			UpperCAmelCase_     : Dict			     =					BlipaConfig(vision_config=A__	,text_config=A__						)

			return config, image_size

@torch.no_grad()
def 				snake_case      (   A__	,A__=None	,A__=False						):
			UpperCAmelCase_     : str			     =					(
			    AutoTokenizer.from_pretrained("facebook/opt-2.7b"						)
			    if "opt" in model_name
			    else AutoTokenizer.from_pretrained("google/flan-t5-xl"						)
			)
			UpperCAmelCase_     : List[str]			     =					tokenizer("\n"	,add_special_tokens=A__						).input_ids[0]
			UpperCAmelCase_     ,	UpperCAmelCase_     : List[str]			     =					get_blipa_config(A__	,eos_token_id=A__						)

			UpperCAmelCase_     : List[Any]			     =					BlipaForConditionalGeneration(A__						).eval()

			UpperCAmelCase_     : Dict			     =					{
			    "blip2-opt-2.7b": ("blip2_opt", "pretrain_opt2.7b"),
			    "blip2-opt-6.7b": ("blip2_opt", "pretrain_opt6.7b"),
			    "blip2-opt-2.7b-coco": ("blip2_opt", "caption_coco_opt2.7b"),
			    "blip2-opt-6.7b-coco": ("blip2_opt", "caption_coco_opt6.7b"),
			    "blip2-flan-t5-xl": ("blip2_t5", "pretrain_flant5xl"),
			    "blip2-flan-t5-xl-coco": ("blip2_t5", "caption_coco_flant5xl"),
			    "blip2-flan-t5-xxl": ("blip2_t5", "pretrain_flant5xxl"),
			}

			UpperCAmelCase_     ,	UpperCAmelCase_     : List[Any]			     =					model_name_to_original[model_name]

			# load original model
			print("Loading original model..."						)
			UpperCAmelCase_     : int			     =					"cuda" if torch.cuda.is_available() else "cpu"
			UpperCAmelCase_     ,	UpperCAmelCase_     ,	UpperCAmelCase_     : List[Any]			     =					load_model_and_preprocess(
			    name=A__	,model_type=A__	,is_eval=A__	,device=A__						)
			original_model.eval()
			print("Done!"						)

			# update state dict keys
			UpperCAmelCase_     : List[Any]			     =					original_model.state_dict()
			UpperCAmelCase_     : List[Any]			     =					create_rename_keys(A__						)
			for src, dest in rename_keys:
						rename_key(A__	,A__	,A__						)

			# some keys can be renamed efficiently
			for key, val in state_dict.copy().items():
						UpperCAmelCase_     : str			     =					state_dict.pop(A__						)
						if key.startswith("Qformer.bert"						):
									UpperCAmelCase_     : Tuple			     =					key.replace("Qformer.bert"	,"qformer"						)
						if "attention.self" in key:
									UpperCAmelCase_     : Optional[Any]			     =					key.replace("self"	,"attention"						)
						if "opt_proj" in key:
									UpperCAmelCase_     : int			     =					key.replace("opt_proj"	,"language_projection"						)
						if "t5_proj" in key:
									UpperCAmelCase_     : Any			     =					key.replace("t5_proj"	,"language_projection"						)
						if key.startswith("opt"						):
									UpperCAmelCase_     : Dict			     =					key.replace("opt"	,"language"						)
						if key.startswith("t5"						):
									UpperCAmelCase_     : Union[str, Any]			     =					key.replace("t5"	,"language"						)
						UpperCAmelCase_     : List[str]			     =					val

			# read in qv biases
			read_in_q_v_bias(A__	,A__						)

			UpperCAmelCase_     ,	UpperCAmelCase_     : Optional[int]			     =					hf_model.load_state_dict(A__	,strict=A__						)
			assert len(A__						) == 0
			assert unexpected_keys == ["qformer.embeddings.position_ids"]

			UpperCAmelCase_     : int			     =					load_demo_image()
			UpperCAmelCase_     : Union[str, Any]			     =					vis_processors["eval"](A__						).unsqueeze(0						).to(A__						)
			UpperCAmelCase_     : Optional[int]			     =					tokenizer(["\n"]	,return_tensors="pt"						).input_ids.to(A__						)

			# create processor
			UpperCAmelCase_     : str			     =					BlipImageProcessor(
			    size={"height": image_size, "width": image_size}	,image_mean=A__	,image_std=A__						)
			UpperCAmelCase_     : List[str]			     =					BlipaProcessor(image_processor=A__	,tokenizer=A__						)
			UpperCAmelCase_     : Tuple			     =					processor(images=A__	,return_tensors="pt"						).pixel_values.to(A__						)

			# make sure processor creates exact same pixel values
			assert torch.allclose(A__	,A__						)

			original_model.to(A__						)
			hf_model.to(A__						)
			with torch.no_grad():
						if "opt" in model_name:
									UpperCAmelCase_     : Optional[int]			     =					original_model({"image": original_pixel_values, "text_input": [""]}						).logits
									UpperCAmelCase_     : List[str]			     =					hf_model(A__	,A__						).logits
						else:
									UpperCAmelCase_     : int			     =					original_model(
									    {"image": original_pixel_values, "text_input": ["\n"], "text_output": ["\n"]}						).logits
									UpperCAmelCase_     : Optional[Any]			     =					input_ids.masked_fill(input_ids == tokenizer.pad_token_id	,-1_00						)
									UpperCAmelCase_     : Optional[int]			     =					hf_model(A__	,A__	,labels=A__						).logits

			assert original_logits.shape == logits.shape
			print("First values of original logits:"	,original_logits[0, :3, :3]						)
			print("First values of HF logits:"	,logits[0, :3, :3]						)

			# assert values
			if model_name == "blip2-flan-t5-xl":
						UpperCAmelCase_     : List[Any]			     =					torch.tensor(
						    [[-41.5850, -4.4440, -8.9922], [-47.4322, -5.9143, -1.7340]]	,device=A__						)
						assert torch.allclose(logits[0, :3, :3]	,A__	,atol=1e-4						)
			elif model_name == "blip2-flan-t5-xl-coco":
						UpperCAmelCase_     : int			     =					torch.tensor(
						    [[-57.0109, -9.8967, -12.6280], [-68.6578, -12.7191, -10.5065]]	,device=A__						)
			else:
						# cast to same type
						UpperCAmelCase_     : Dict			     =					logits.dtype
						assert torch.allclose(original_logits.to(A__						)	,A__	,atol=1e-2						)
			print("Looks ok!"						)

			print("Generating a caption..."						)
			UpperCAmelCase_     : str			     =					""
			UpperCAmelCase_     : str			     =					tokenizer(A__	,return_tensors="pt"						).input_ids.to(A__						)

			UpperCAmelCase_     : str			     =					original_model.generate({"image": original_pixel_values}						)
			UpperCAmelCase_     : List[Any]			     =					hf_model.generate(
			    A__	,A__	,do_sample=A__	,num_beams=5	,max_length=30	,min_length=1	,top_p=0.9	,repetition_penalty=1.0	,length_penalty=1.0	,temperature=1	,)
			print("Original generation:"	,A__						)
			UpperCAmelCase_     : Optional[int]			     =					input_ids.shape[1]
			UpperCAmelCase_     : Union[str, Any]			     =					processor.batch_decode(outputs[:, prompt_length:]	,skip_special_tokens=A__						)
			UpperCAmelCase_     : List[Any]			     =					[text.strip() for text in output_text]
			print("HF generation:"	,A__						)

			if pytorch_dump_folder_path is not None:
						processor.save_pretrained(A__						)
						hf_model.save_pretrained(A__						)

			if push_to_hub:
						processor.push_to_hub(F"""nielsr/{model_name}"""						)
						hf_model.push_to_hub(F"""nielsr/{model_name}"""						)


if __name__ == "__main__":
			lowerCamelCase_					=      argparse.ArgumentParser()
			lowerCamelCase_					=      [
			    '''blip2-opt-2.7b''',
			    '''blip2-opt-6.7b''',
			    '''blip2-opt-2.7b-coco''',
			    '''blip2-opt-6.7b-coco''',
			    '''blip2-flan-t5-xl''',
			    '''blip2-flan-t5-xl-coco''',
			    '''blip2-flan-t5-xxl''',
			]
			parser.add_argument(
			    '''--model_name''',
			    default='''blip2-opt-2.7b''',
			    choices=choices,
			    type=str,
			    help='''Path to hf config.json of model to convert''',
			)
			parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
			parser.add_argument(
			    '''--push_to_hub''',
			    action='''store_true''',
			    help='''Whether to push the model and processor to the hub after converting''',
			)

			lowerCamelCase_					=      parser.parse_args()

			convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)











"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_					=      logging.get_logger(__name__)

lowerCamelCase_					=      {
    '''RWKV/rwkv-4-169m-pile''': '''https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-430m-pile''': '''https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-1b5-pile''': '''https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-3b-pile''': '''https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-7b-pile''': '''https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-14b-pile''': '''https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-raven-1b5''': '''https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json''',
    '''RWKV/rwkv-raven-3b''': '''https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-7b''': '''https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-14b''': '''https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json''',
}







class 			UpperCamelCase_       (__A     ):
	__magic_name__	      =				'''rwkv'''
	__magic_name__	      =				{'''max_position_embeddings''': '''context_length'''}





	def __init__(      self					: str	,	lowerCAmelCase_					: str=50_277	,	lowerCAmelCase_					: Optional[int]=1_024	,	lowerCAmelCase_					: Optional[int]=4_096	,	lowerCAmelCase_					: Union[str, Any]=32	,	lowerCAmelCase_					: List[Any]=None	,	lowerCAmelCase_					: int=None	,	lowerCAmelCase_					: List[Any]=1e-5	,	lowerCAmelCase_					: str=0	,	lowerCAmelCase_					: Tuple=0	,	lowerCAmelCase_					: int=6	,	lowerCAmelCase_					: Optional[Any]=False	,	lowerCAmelCase_					: Any=True	,	**lowerCAmelCase_					: List[Any]	,	)						->			List[str]:
				UpperCAmelCase_     : Tuple			     =					vocab_size
				UpperCAmelCase_     : List[str]			     =					context_length
				UpperCAmelCase_     : Dict			     =					hidden_size
				UpperCAmelCase_     : Optional[int]			     =					num_hidden_layers
				UpperCAmelCase_     : Optional[int]			     =					attention_hidden_size if attention_hidden_size is not None else hidden_size
				UpperCAmelCase_     : Dict			     =					intermediate_size if intermediate_size is not None else 4 * hidden_size
				UpperCAmelCase_     : Any			     =					layer_norm_epsilon
				UpperCAmelCase_     : List[Any]			     =					rescale_every
				UpperCAmelCase_     : List[str]			     =					use_cache

				UpperCAmelCase_     : List[str]			     =					bos_token_id
				UpperCAmelCase_     : Union[str, Any]			     =					eos_token_id

				super().__init__(
				    tie_word_embeddings=lowerCAmelCase_	,	bos_token_id=lowerCAmelCase_	,	eos_token_id=lowerCAmelCase_	,	**lowerCAmelCase_						)



