





import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD

def   a_    ( ):
   __lowerCAmelCase								=	'https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png'
   __lowerCAmelCase								=	Image.open(requests.get(lowerCAmelCase_,	stream=lowerCAmelCase_	).raw	).convert('RGB'	)

   return image

def   a_    ( lowerCAmelCase_      :			Optional[int]	):
   __lowerCAmelCase								=	[]
   # fmt: off

   # vision encoder
   rename_keys.append(('visual_encoder.cls_token', 'vision_model.embeddings.class_embedding')	)
   rename_keys.append(('visual_encoder.pos_embed', 'vision_model.embeddings.position_embedding')	)
   rename_keys.append(('visual_encoder.patch_embed.proj.weight', 'vision_model.embeddings.patch_embedding.weight')	)
   rename_keys.append(('visual_encoder.patch_embed.proj.bias', 'vision_model.embeddings.patch_embedding.bias')	)
   rename_keys.append(('ln_vision.weight', 'vision_model.post_layernorm.weight')	)
   rename_keys.append(('ln_vision.bias', 'vision_model.post_layernorm.bias')	)

   for i in range(config.vision_config.num_hidden_layers	):
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.weight""", F"""vision_model.encoder.layers.{i}.layer_norm1.weight""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.bias""", F"""vision_model.encoder.layers.{i}.layer_norm1.bias""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.weight""", F"""vision_model.encoder.layers.{i}.layer_norm2.weight""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.bias""", F"""vision_model.encoder.layers.{i}.layer_norm2.bias""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.qkv.weight""", F"""vision_model.encoder.layers.{i}.self_attn.qkv.weight""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.weight""", F"""vision_model.encoder.layers.{i}.self_attn.projection.weight""",)	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.bias""", F"""vision_model.encoder.layers.{i}.self_attn.projection.bias""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc1.weight""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc1.bias""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc2.weight""")	)
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc2.bias""")	)

   # QFormer
   rename_keys.append(('Qformer.bert.embeddings.LayerNorm.weight', 'qformer.layernorm.weight')	)
   rename_keys.append(('Qformer.bert.embeddings.LayerNorm.bias', 'qformer.layernorm.bias')	)

   # fmt: on
   return rename_keys

def   a_    ( lowerCAmelCase_      :			Optional[int],	lowerCAmelCase_      :			str,	lowerCAmelCase_      :			Union[str, Any]	):
   __lowerCAmelCase								=	dct.pop(lowerCAmelCase_	)
   __lowerCAmelCase								=	val

def   a_    ( lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			Optional[Any]	):
   for i in range(config.vision_config.num_hidden_layers	):
      # read in original q and v biases
      __lowerCAmelCase								=	state_dict.pop(F"""visual_encoder.blocks.{i}.attn.q_bias"""	)
      __lowerCAmelCase								=	state_dict.pop(F"""visual_encoder.blocks.{i}.attn.v_bias"""	)

      # next, set bias in the state dict
      __lowerCAmelCase								=	torch.cat((q_bias, torch.zeros_like(lowerCAmelCase_,	requires_grad=lowerCAmelCase_	), v_bias)	)
      __lowerCAmelCase								=	qkv_bias

def   a_    ( lowerCAmelCase_      :			Dict,	lowerCAmelCase_      :			Any	):
   __lowerCAmelCase								=	364 if 'coco' in model_name else 224
   __lowerCAmelCase								=	BlipaVisionConfig(image_size=lowerCAmelCase_	).to_dict()

   # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
   # seems like flan-T5 models don't have bos_token_id properly set?
   if "opt-2.7b" in model_name:
      __lowerCAmelCase								=	OPTConfig.from_pretrained('facebook/opt-2.7b',	eos_token_id=lowerCAmelCase_	).to_dict()
   elif "opt-6.7b" in model_name:
      __lowerCAmelCase								=	OPTConfig.from_pretrained('facebook/opt-6.7b',	eos_token_id=lowerCAmelCase_	).to_dict()
   elif "t5-xl" in model_name:
      __lowerCAmelCase								=	TaConfig.from_pretrained('google/flan-t5-xl',	dense_act_fn='gelu',	bos_token_id=1	).to_dict()
   elif "t5-xxl" in model_name:
      __lowerCAmelCase								=	TaConfig.from_pretrained('google/flan-t5-xxl',	dense_act_fn='gelu',	bos_token_id=1	).to_dict()

   __lowerCAmelCase								=	BlipaConfig(vision_config=lowerCAmelCase_,	text_config=lowerCAmelCase_	)

   return config, image_size

@torch.no_grad()
def   a_    ( lowerCAmelCase_      :			Union[str, Any],	lowerCAmelCase_      :			Optional[int]=None,	lowerCAmelCase_      :			int=False	):
   __lowerCAmelCase								=	(
       AutoTokenizer.from_pretrained('facebook/opt-2.7b'	)
       if 'opt' in model_name
       else AutoTokenizer.from_pretrained('google/flan-t5-xl'	)
   )
   __lowerCAmelCase								=	tokenizer('\n',	add_special_tokens=lowerCAmelCase_	).input_ids[0]
   __lowerCAmelCase	,	__lowerCAmelCase								=	get_blipa_config(lowerCAmelCase_,	eos_token_id=lowerCAmelCase_	)

   __lowerCAmelCase								=	BlipaForConditionalGeneration(lowerCAmelCase_	).eval()

   __lowerCAmelCase								=	{
       'blip2-opt-2.7b': ('blip2_opt', 'pretrain_opt2.7b'),
       'blip2-opt-6.7b': ('blip2_opt', 'pretrain_opt6.7b'),
       'blip2-opt-2.7b-coco': ('blip2_opt', 'caption_coco_opt2.7b'),
       'blip2-opt-6.7b-coco': ('blip2_opt', 'caption_coco_opt6.7b'),
       'blip2-flan-t5-xl': ('blip2_t5', 'pretrain_flant5xl'),
       'blip2-flan-t5-xl-coco': ('blip2_t5', 'caption_coco_flant5xl'),
       'blip2-flan-t5-xxl': ('blip2_t5', 'pretrain_flant5xxl'),
   }

   __lowerCAmelCase	,	__lowerCAmelCase								=	model_name_to_original[model_name]

   # load original model
   print('Loading original model...'	)
   __lowerCAmelCase								=	'cuda' if torch.cuda.is_available() else 'cpu'
   __lowerCAmelCase	,	__lowerCAmelCase	,	__lowerCAmelCase								=	load_model_and_preprocess(
       name=lowerCAmelCase_,	model_type=lowerCAmelCase_,	is_eval=lowerCAmelCase_,	device=lowerCAmelCase_	)
   original_model.eval()
   print('Done!'	)

   # update state dict keys
   __lowerCAmelCase								=	original_model.state_dict()
   __lowerCAmelCase								=	create_rename_keys(lowerCAmelCase_	)
   for src, dest in rename_keys:
      rename_key(lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_	)

   # some keys can be renamed efficiently
   for key, val in state_dict.copy().items():
      __lowerCAmelCase								=	state_dict.pop(lowerCAmelCase_	)
      if key.startswith('Qformer.bert'	):
         __lowerCAmelCase								=	key.replace('Qformer.bert',	'qformer'	)
      if "attention.self" in key:
         __lowerCAmelCase								=	key.replace('self',	'attention'	)
      if "opt_proj" in key:
         __lowerCAmelCase								=	key.replace('opt_proj',	'language_projection'	)
      if "t5_proj" in key:
         __lowerCAmelCase								=	key.replace('t5_proj',	'language_projection'	)
      if key.startswith('opt'	):
         __lowerCAmelCase								=	key.replace('opt',	'language'	)
      if key.startswith('t5'	):
         __lowerCAmelCase								=	key.replace('t5',	'language'	)
      __lowerCAmelCase								=	val

   # read in qv biases
   read_in_q_v_bias(lowerCAmelCase_,	lowerCAmelCase_	)

   __lowerCAmelCase	,	__lowerCAmelCase								=	hf_model.load_state_dict(lowerCAmelCase_,	strict=lowerCAmelCase_	)
   assert len(lowerCAmelCase_	) == 0
   assert unexpected_keys == ["qformer.embeddings.position_ids"]

   __lowerCAmelCase								=	load_demo_image()
   __lowerCAmelCase								=	vis_processors['eval'](lowerCAmelCase_	).unsqueeze(0	).to(lowerCAmelCase_	)
   __lowerCAmelCase								=	tokenizer(['\n'],	return_tensors='pt'	).input_ids.to(lowerCAmelCase_	)

   # create processor
   __lowerCAmelCase								=	BlipImageProcessor(
       size={'height': image_size, 'width': image_size},	image_mean=lowerCAmelCase_,	image_std=lowerCAmelCase_	)
   __lowerCAmelCase								=	BlipaProcessor(image_processor=lowerCAmelCase_,	tokenizer=lowerCAmelCase_	)
   __lowerCAmelCase								=	processor(images=lowerCAmelCase_,	return_tensors='pt'	).pixel_values.to(lowerCAmelCase_	)

   # make sure processor creates exact same pixel values
   assert torch.allclose(lowerCAmelCase_,	lowerCAmelCase_	)

   original_model.to(lowerCAmelCase_	)
   hf_model.to(lowerCAmelCase_	)
   with torch.no_grad():
      if "opt" in model_name:
         __lowerCAmelCase								=	original_model({'image': original_pixel_values, 'text_input': ['']}	).logits
         __lowerCAmelCase								=	hf_model(lowerCAmelCase_,	lowerCAmelCase_	).logits
      else:
         __lowerCAmelCase								=	original_model(
             {'image': original_pixel_values, 'text_input': ['\n'], 'text_output': ['\n']}	).logits
         __lowerCAmelCase								=	input_ids.masked_fill(input_ids == tokenizer.pad_token_id,	-100	)
         __lowerCAmelCase								=	hf_model(lowerCAmelCase_,	lowerCAmelCase_,	labels=lowerCAmelCase_	).logits

   assert original_logits.shape == logits.shape
   print('First values of original logits:',	original_logits[0, :3, :3]	)
   print('First values of HF logits:',	logits[0, :3, :3]	)

   # assert values
   if model_name == "blip2-flan-t5-xl":
      __lowerCAmelCase								=	torch.tensor(
          [[-41.5850, -4.4440, -8.9922], [-47.4322, -5.9143, -1.7340]],	device=lowerCAmelCase_	)
      assert torch.allclose(logits[0, :3, :3],	lowerCAmelCase_,	atol=1E-4	)
   elif model_name == "blip2-flan-t5-xl-coco":
      __lowerCAmelCase								=	torch.tensor(
          [[-57.0109, -9.8967, -12.6280], [-68.6578, -12.7191, -10.5065]],	device=lowerCAmelCase_	)
   else:
      # cast to same type
      __lowerCAmelCase								=	logits.dtype
      assert torch.allclose(original_logits.to(lowerCAmelCase_	),	lowerCAmelCase_,	atol=1E-2	)
   print('Looks ok!'	)

   print('Generating a caption...'	)
   __lowerCAmelCase								=	''
   __lowerCAmelCase								=	tokenizer(lowerCAmelCase_,	return_tensors='pt'	).input_ids.to(lowerCAmelCase_	)

   __lowerCAmelCase								=	original_model.generate({'image': original_pixel_values}	)
   __lowerCAmelCase								=	hf_model.generate(
       lowerCAmelCase_,	lowerCAmelCase_,	do_sample=lowerCAmelCase_,	num_beams=5,	max_length=30,	min_length=1,	top_p=0.9,	repetition_penalty=1.0,	length_penalty=1.0,	temperature=1,	)
   print('Original generation:',	lowerCAmelCase_	)
   __lowerCAmelCase								=	input_ids.shape[1]
   __lowerCAmelCase								=	processor.batch_decode(outputs[:, prompt_length:],	skip_special_tokens=lowerCAmelCase_	)
   __lowerCAmelCase								=	[text.strip() for text in output_text]
   print('HF generation:',	lowerCAmelCase_	)

   if pytorch_dump_folder_path is not None:
      processor.save_pretrained(lowerCAmelCase_	)
      hf_model.save_pretrained(lowerCAmelCase_	)

   if push_to_hub:
      processor.push_to_hub(F"""nielsr/{model_name}"""	)
      hf_model.push_to_hub(F"""nielsr/{model_name}"""	)


if __name__ == "__main__":
     _snake_case  :    Tuple						    =  argparse.ArgumentParser()
     _snake_case  :    Optional[Any]						    =  [
         'blip2-opt-2.7b',
         'blip2-opt-6.7b',
         'blip2-opt-2.7b-coco',
         'blip2-opt-6.7b-coco',
         'blip2-flan-t5-xl',
         'blip2-flan-t5-xl-coco',
         'blip2-flan-t5-xxl',
     ]
     parser.add_argument(
         '--model_name',
         default='blip2-opt-2.7b',
         choices=choices,
         type=str,
         help='Path to hf config.json of model to convert',
     )
     parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
     parser.add_argument(
         '--push_to_hub',
         action='store_true',
         help='Whether to push the model and processor to the hub after converting',
     )

     _snake_case  :    Optional[Any]						    =  parser.parse_args()

     convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)








from ..utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_scipy_available,
    is_torch_available,
    is_torchsde_available,
)


try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_pt_objects import *  # noqa F403
else:
     from .scheduling_consistency_models import CMStochasticIterativeScheduler
     from .scheduling_ddim import DDIMScheduler
     from .scheduling_ddim_inverse import DDIMInverseScheduler
     from .scheduling_ddim_parallel import DDIMParallelScheduler
     from .scheduling_ddpm import DDPMScheduler
     from .scheduling_ddpm_parallel import DDPMParallelScheduler
     from .scheduling_deis_multistep import DEISMultistepScheduler
     from .scheduling_dpmsolver_multistep import DPMSolverMultistepScheduler
     from .scheduling_dpmsolver_multistep_inverse import DPMSolverMultistepInverseScheduler
     from .scheduling_dpmsolver_singlestep import DPMSolverSinglestepScheduler
     from .scheduling_euler_ancestral_discrete import EulerAncestralDiscreteScheduler
     from .scheduling_euler_discrete import EulerDiscreteScheduler
     from .scheduling_heun_discrete import HeunDiscreteScheduler
     from .scheduling_ipndm import IPNDMScheduler
     from .scheduling_k_dpm_2_ancestral_discrete import KDPMaAncestralDiscreteScheduler
     from .scheduling_k_dpm_2_discrete import KDPMaDiscreteScheduler
     from .scheduling_karras_ve import KarrasVeScheduler
     from .scheduling_pndm import PNDMScheduler
     from .scheduling_repaint import RePaintScheduler
     from .scheduling_sde_ve import ScoreSdeVeScheduler
     from .scheduling_sde_vp import ScoreSdeVpScheduler
     from .scheduling_unclip import UnCLIPScheduler
     from .scheduling_unipc_multistep import UniPCMultistepScheduler
     from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin
     from .scheduling_vq_diffusion import VQDiffusionScheduler

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_flax_objects import *  # noqa F403
else:
     from .scheduling_ddim_flax import FlaxDDIMScheduler
     from .scheduling_ddpm_flax import FlaxDDPMScheduler
     from .scheduling_dpmsolver_multistep_flax import FlaxDPMSolverMultistepScheduler
     from .scheduling_karras_ve_flax import FlaxKarrasVeScheduler
     from .scheduling_lms_discrete_flax import FlaxLMSDiscreteScheduler
     from .scheduling_pndm_flax import FlaxPNDMScheduler
     from .scheduling_sde_ve_flax import FlaxScoreSdeVeScheduler
     from .scheduling_utils_flax import (
         FlaxKarrasDiffusionSchedulers,
         FlaxSchedulerMixin,
         FlaxSchedulerOutput,
         broadcast_to_shape_from_left,
     )


try:
     if not (is_torch_available() and is_scipy_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_scipy_objects import *  # noqa F403
else:
     from .scheduling_lms_discrete import LMSDiscreteScheduler

try:
     if not (is_torch_available() and is_torchsde_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_torchsde_objects import *  # noqa F403
else:
     from .scheduling_dpmsolver_sde import DPMSolverSDEScheduler

