




import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD





def __UpperCamelCase			(       ):
						lowerCAmelCase_      			=  "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png"
						lowerCAmelCase_      			=  Image.open(requests.get(_snake_case ,  stream=_snake_case  ).raw  ).convert('''RGB'''  )

						return image





def __UpperCamelCase			(       _A  ):
						lowerCAmelCase_      			=  []
						# fmt: off

						# vision encoder
						rename_keys.append(('''visual_encoder.cls_token''', '''vision_model.embeddings.class_embedding''')  )
						rename_keys.append(('''visual_encoder.pos_embed''', '''vision_model.embeddings.position_embedding''')  )
						rename_keys.append(('''visual_encoder.patch_embed.proj.weight''', '''vision_model.embeddings.patch_embedding.weight''')  )
						rename_keys.append(('''visual_encoder.patch_embed.proj.bias''', '''vision_model.embeddings.patch_embedding.bias''')  )
						rename_keys.append(('''ln_vision.weight''', '''vision_model.post_layernorm.weight''')  )
						rename_keys.append(('''ln_vision.bias''', '''vision_model.post_layernorm.bias''')  )

						for i in range(config.vision_config.num_hidden_layers  ):
												rename_keys.append((f"visual_encoder.blocks.{i}.norm1.weight", f"vision_model.encoder.layers.{i}.layer_norm1.weight")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.norm1.bias", f"vision_model.encoder.layers.{i}.layer_norm1.bias")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.norm2.weight", f"vision_model.encoder.layers.{i}.layer_norm2.weight")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.norm2.bias", f"vision_model.encoder.layers.{i}.layer_norm2.bias")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.attn.qkv.weight", f"vision_model.encoder.layers.{i}.self_attn.qkv.weight")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.attn.proj.weight", f"vision_model.encoder.layers.{i}.self_attn.projection.weight",)  )
												rename_keys.append((f"visual_encoder.blocks.{i}.attn.proj.bias", f"vision_model.encoder.layers.{i}.self_attn.projection.bias")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc1.weight", f"vision_model.encoder.layers.{i}.mlp.fc1.weight")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc1.bias", f"vision_model.encoder.layers.{i}.mlp.fc1.bias")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc2.weight", f"vision_model.encoder.layers.{i}.mlp.fc2.weight")  )
												rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc2.bias", f"vision_model.encoder.layers.{i}.mlp.fc2.bias")  )

						# QFormer
						rename_keys.append(('''Qformer.bert.embeddings.LayerNorm.weight''', '''qformer.layernorm.weight''')  )
						rename_keys.append(('''Qformer.bert.embeddings.LayerNorm.bias''', '''qformer.layernorm.bias''')  )

						# fmt: on
						return rename_keys





def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  dct.pop(_snake_case  )
						lowerCAmelCase_      			=  val





def __UpperCamelCase			(       _A ,  _A  ):


						for i in range(config.vision_config.num_hidden_layers  ):
												# read in original q and v biases
												lowerCAmelCase_      			=  state_dict.pop(f"visual_encoder.blocks.{i}.attn.q_bias"  )
												lowerCAmelCase_      			=  state_dict.pop(f"visual_encoder.blocks.{i}.attn.v_bias"  )

												# next, set bias in the state dict
												lowerCAmelCase_      			=  torch.cat((q_bias, torch.zeros_like(_snake_case ,  requires_grad=_snake_case  ), v_bias)  )
												lowerCAmelCase_      			=  qkv_bias





def __UpperCamelCase			(       _A ,  _A  ):
						lowerCAmelCase_      			=  364 if "coco" in model_name else 224
						lowerCAmelCase_      			=  BlipaVisionConfig(image_size=_snake_case  ).to_dict()

						# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
						# seems like flan-T5 models don't have bos_token_id properly set?
						if "opt-2.7b" in model_name:
												lowerCAmelCase_      			=  OPTConfig.from_pretrained('''facebook/opt-2.7b''' ,  eos_token_id=_snake_case  ).to_dict()
						elif "opt-6.7b" in model_name:
												lowerCAmelCase_      			=  OPTConfig.from_pretrained('''facebook/opt-6.7b''' ,  eos_token_id=_snake_case  ).to_dict()
						elif "t5-xl" in model_name:
												lowerCAmelCase_      			=  TaConfig.from_pretrained('''google/flan-t5-xl''' ,  dense_act_fn='''gelu''' ,  bos_token_id=1  ).to_dict()
						elif "t5-xxl" in model_name:
												lowerCAmelCase_      			=  TaConfig.from_pretrained('''google/flan-t5-xxl''' ,  dense_act_fn='''gelu''' ,  bos_token_id=1  ).to_dict()

						lowerCAmelCase_      			=  BlipaConfig(vision_config=_snake_case ,  text_config=_snake_case  )

						return config, image_size





@torch.no_grad()
def __UpperCamelCase			(       _A ,  _A=None ,  _A=False  ):
						lowerCAmelCase_      			=  (
						    AutoTokenizer.from_pretrained('''facebook/opt-2.7b'''  )
						    if "opt" in model_name
						    else AutoTokenizer.from_pretrained('''google/flan-t5-xl'''  )
						)
						lowerCAmelCase_      			=  tokenizer('''\n''' ,  add_special_tokens=_snake_case  ).input_ids[0]
						lowerCAmelCase_      			=  get_blipa_config(_snake_case ,  eos_token_id=_snake_case  )

						lowerCAmelCase_      			=  BlipaForConditionalGeneration(_snake_case  ).eval()

						lowerCAmelCase_      			=  {
						    "blip2-opt-2.7b": ("blip2_opt", "pretrain_opt2.7b"),
						    "blip2-opt-6.7b": ("blip2_opt", "pretrain_opt6.7b"),
						    "blip2-opt-2.7b-coco": ("blip2_opt", "caption_coco_opt2.7b"),
						    "blip2-opt-6.7b-coco": ("blip2_opt", "caption_coco_opt6.7b"),
						    "blip2-flan-t5-xl": ("blip2_t5", "pretrain_flant5xl"),
						    "blip2-flan-t5-xl-coco": ("blip2_t5", "caption_coco_flant5xl"),
						    "blip2-flan-t5-xxl": ("blip2_t5", "pretrain_flant5xxl"),
						}

						lowerCAmelCase_      			=  model_name_to_original[model_name]

						# load original model
						print('''Loading original model...'''  )
						lowerCAmelCase_      			=  "cuda" if torch.cuda.is_available() else "cpu"
						lowerCAmelCase_      			=  load_model_and_preprocess(
						    name=_snake_case ,  model_type=_snake_case ,  is_eval=_snake_case ,  device=_snake_case  )
						original_model.eval()
						print('''Done!'''  )

						# update state dict keys
						lowerCAmelCase_      			=  original_model.state_dict()
						lowerCAmelCase_      			=  create_rename_keys(_snake_case  )
						for src, dest in rename_keys:
												rename_key(_snake_case ,  _snake_case ,  _snake_case  )

						# some keys can be renamed efficiently
						for key, val in state_dict.copy().items():
												lowerCAmelCase_      			=  state_dict.pop(_snake_case  )
												if key.startswith('''Qformer.bert'''  ):
																		lowerCAmelCase_      			=  key.replace('''Qformer.bert''' ,  '''qformer'''  )
												if "attention.self" in key:
																		lowerCAmelCase_      			=  key.replace('''self''' ,  '''attention'''  )
												if "opt_proj" in key:
																		lowerCAmelCase_      			=  key.replace('''opt_proj''' ,  '''language_projection'''  )
												if "t5_proj" in key:
																		lowerCAmelCase_      			=  key.replace('''t5_proj''' ,  '''language_projection'''  )
												if key.startswith('''opt'''  ):
																		lowerCAmelCase_      			=  key.replace('''opt''' ,  '''language'''  )
												if key.startswith('''t5'''  ):
																		lowerCAmelCase_      			=  key.replace('''t5''' ,  '''language'''  )
												lowerCAmelCase_      			=  val

						# read in qv biases
						read_in_q_v_bias(_snake_case ,  _snake_case  )

						lowerCAmelCase_      			=  hf_model.load_state_dict(_snake_case ,  strict=_snake_case  )
						assert len(_snake_case  ) == 0
						assert unexpected_keys == ["qformer.embeddings.position_ids"]

						lowerCAmelCase_      			=  load_demo_image()
						lowerCAmelCase_      			=  vis_processors["eval"](_snake_case  ).unsqueeze(0  ).to(_snake_case  )
						lowerCAmelCase_      			=  tokenizer(['''\n'''] ,  return_tensors='''pt'''  ).input_ids.to(_snake_case  )

						# create processor
						lowerCAmelCase_      			=  BlipImageProcessor(
						    size={'''height''': image_size, '''width''': image_size} ,  image_mean=_snake_case ,  image_std=_snake_case  )
						lowerCAmelCase_      			=  BlipaProcessor(image_processor=_snake_case ,  tokenizer=_snake_case  )
						lowerCAmelCase_      			=  processor(images=_snake_case ,  return_tensors='''pt'''  ).pixel_values.to(_snake_case  )

						# make sure processor creates exact same pixel values
						assert torch.allclose(_snake_case ,  _snake_case  )

						original_model.to(_snake_case  )
						hf_model.to(_snake_case  )
						with torch.no_grad():
												if "opt" in model_name:
																		lowerCAmelCase_      			=  original_model({'''image''': original_pixel_values, '''text_input''': ['''''']}  ).logits
																		lowerCAmelCase_      			=  hf_model(_snake_case ,  _snake_case  ).logits
												else:
																		lowerCAmelCase_      			=  original_model(
																		    {'''image''': original_pixel_values, '''text_input''': ['''\n'''], '''text_output''': ['''\n''']}  ).logits
																		lowerCAmelCase_      			=  input_ids.masked_fill(input_ids == tokenizer.pad_token_id ,  -100  )
																		lowerCAmelCase_      			=  hf_model(_snake_case ,  _snake_case ,  labels=_snake_case  ).logits

						assert original_logits.shape == logits.shape
						print('''First values of original logits:''' ,  original_logits[0, :3, :3]  )
						print('''First values of HF logits:''' ,  logits[0, :3, :3]  )

						# assert values
						if model_name == "blip2-flan-t5-xl":
												lowerCAmelCase_      			=  torch.tensor(
												    [[-4_1.5_8_5_0, -4.4_4_4_0, -8.9_9_2_2], [-4_7.4_3_2_2, -5.9_1_4_3, -1.7_3_4_0]] ,  device=_snake_case  )
												assert torch.allclose(logits[0, :3, :3] ,  _snake_case ,  atol=1E-4  )
						elif model_name == "blip2-flan-t5-xl-coco":
												lowerCAmelCase_      			=  torch.tensor(
												    [[-5_7.0_1_0_9, -9.8_9_6_7, -1_2.6_2_8_0], [-6_8.6_5_7_8, -1_2.7_1_9_1, -1_0.5_0_6_5]] ,  device=_snake_case  )
						else:
												# cast to same type
												lowerCAmelCase_      			=  logits.dtype
												assert torch.allclose(original_logits.to(_snake_case  ) ,  _snake_case ,  atol=1E-2  )
						print('''Looks ok!'''  )

						print('''Generating a caption...'''  )
						lowerCAmelCase_      			=  ""
						lowerCAmelCase_      			=  tokenizer(_snake_case ,  return_tensors='''pt'''  ).input_ids.to(_snake_case  )

						lowerCAmelCase_      			=  original_model.generate({'''image''': original_pixel_values}  )
						lowerCAmelCase_      			=  hf_model.generate(
						    _snake_case ,  _snake_case ,  do_sample=_snake_case ,  num_beams=5 ,  max_length=30 ,  min_length=1 ,  top_p=0.9 ,  repetition_penalty=1.0 ,  length_penalty=1.0 ,  temperature=1 ,  )
						print('''Original generation:''' ,  _snake_case  )
						lowerCAmelCase_      			=  input_ids.shape[1]
						lowerCAmelCase_      			=  processor.batch_decode(outputs[:, prompt_length:] ,  skip_special_tokens=_snake_case  )
						lowerCAmelCase_      			=  [text.strip() for text in output_text]
						print('''HF generation:''' ,  _snake_case  )

						if pytorch_dump_folder_path is not None:
												processor.save_pretrained(_snake_case  )
												hf_model.save_pretrained(_snake_case  )

						if push_to_hub:
												processor.push_to_hub(f"nielsr/{model_name}"  )
												hf_model.push_to_hub(f"nielsr/{model_name}"  )


if __name__ == "__main__":
						_A     =			argparse.ArgumentParser()
						_A     =			[
						    "blip2-opt-2.7b",
						    "blip2-opt-6.7b",
						    "blip2-opt-2.7b-coco",
						    "blip2-opt-6.7b-coco",
						    "blip2-flan-t5-xl",
						    "blip2-flan-t5-xl-coco",
						    "blip2-flan-t5-xxl",
						]
						parser.add_argument(
						    '''--model_name''',
						    default='''blip2-opt-2.7b''',
						    choices=choices,
						    type=str,
						    help='''Path to hf config.json of model to convert''',
						)
						parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
						parser.add_argument(
						    '''--push_to_hub''',
						    action='''store_true''',
						    help='''Whether to push the model and processor to the hub after converting''',
						)

						_A     =			parser.parse_args()

						convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)






import absl  # noqa: F401 # Here to have a nice missing dependency error message early on
import nltk  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import six  # noqa: F401 # Here to have a nice missing dependency error message early on
from rouge_score import rouge_scorer, scoring

import datasets


snake_case :		List[Any]							      =  "\\n@inproceedings{lin-2004-rouge,\n    title = \"{ROUGE}: A Package for Automatic Evaluation of Summaries\",\n    author = \"Lin, Chin-Yew\",\n    booktitle = \"Text Summarization Branches Out\",\n    month = jul,\n    year = \"2004\",\n    address = \"Barcelona, Spain\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://www.aclweb.org/anthology/W04-1013\",\n    pages = \"74--81\",\n}\n"

snake_case :		Any							      =  "\\nROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for\nevaluating automatic summarization and machine translation software in natural language processing.\nThe metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.\n\nNote that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.\n\nThis metrics is a wrapper around Google Research reimplementation of ROUGE:\nhttps://github.com/google-research/google-research/tree/master/rouge\n"

snake_case :		str							      =  "\nCalculates average rouge scores for a list of hypotheses and references\nArgs:\n    predictions: list of predictions to score. Each prediction\n        should be a string with tokens separated by spaces.\n    references: list of reference for each prediction. Each\n        reference should be a string with tokens separated by spaces.\n    rouge_types: A list of rouge types to calculate.\n        Valid names:\n        `\"rouge{n}\"` (e.g. `\"rouge1\"`, `\"rouge2\"`) where: {n} is the n-gram based scoring,\n        `\"rougeL\"`: Longest common subsequence based scoring.\n        `\"rougeLSum\"`: rougeLsum splits text using `\"\n\"`.\n        See details in https://github.com/huggingface/datasets/issues/617\n    use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes.\n    use_aggregator: Return aggregates if this is set to True\nReturns:\n    rouge1: rouge_1 (precision, recall, f1),\n    rouge2: rouge_2 (precision, recall, f1),\n    rougeL: rouge_l (precision, recall, f1),\n    rougeLsum: rouge_lsum (precision, recall, f1)\nExamples:\n\n    >>> rouge = datasets.load_metric('rouge')\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> results = rouge.compute(predictions=predictions, references=references)\n    >>> print(list(results.keys()))\n    ['rouge1', 'rouge2', 'rougeL', 'rougeLsum']\n    >>> print(results[\"rouge1\"])\n    AggregateScore(low=Score(precision=1.0, recall=1.0, fmeasure=1.0), mid=Score(precision=1.0, recall=1.0, fmeasure=1.0), high=Score(precision=1.0, recall=1.0, fmeasure=1.0))\n    >>> print(results[\"rouge1\"].mid.fmeasure)\n    1.0\n"





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,						_KWARGS_DESCRIPTION				)
class       _snake_case					(     datasets.Metric				):







			def 			SCREAMING_SNAKE_CASE    (				self		):
				return datasets.MetricInfo(
				    description=_DESCRIPTION						,   citation=_CITATION						,   inputs_description=_KWARGS_DESCRIPTION						,   features=datasets.Features(
				        {
				            "predictions": datasets.Value("string"						,   id="sequence"		),
				            "references": datasets.Value("string"						,   id="sequence"		),
				        }		)						,   codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"]						,   reference_urls=[
				        "https://en.wikipedia.org/wiki/ROUGE_(metric)",
				        "https://github.com/google-research/google-research/tree/master/rouge",
				    ]						,   )







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a						,   _a						,   _a=None						,   _a=True						,   _a=False		):
				if rouge_types is None:
					__magic_name__		:		str	   =     ["rouge1", "rouge2", "rougeL", "rougeLsum"]

				__magic_name__		:		List[str]	   =     rouge_scorer.RougeScorer(rouge_types=_a						,   use_stemmer=_a		)
				if use_aggregator:
					__magic_name__		:		Dict	   =     scoring.BootstrapAggregator()
				else:
					__magic_name__		:		str	   =     []

				for ref, pred in zip(_a						,   _a		):
					__magic_name__		:		Union[str, Any]	   =     scorer.score(_a						,   _a		)
					if use_aggregator:
						aggregator.add_scores(_a		)
					else:
						scores.append(_a		)

				if use_aggregator:
					__magic_name__		:		Any	   =     aggregator.aggregate()
				else:
					__magic_name__		:		List[Any]	   =     {}
					for key in scores[0]:
						__magic_name__		:		str	   =     [score[key] for score in scores]

				return result
