



'''simple docstring'''




import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
a   :			Any					  =    logging.get_logger(__name__)



def       __lowerCamelCase  (					_lowercase	,				_lowercase=False	,				_lowercase=False	,				_lowercase=False		)      ->			int:
							UpperCAmelCase     :		Dict							     =		[]
							for i in range(config.num_hidden_layers		):
														# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
														rename_keys.append((F'''transformer.blocks.{i}.norm1.weight''', F'''vilt.encoder.layer.{i}.layernorm_before.weight''')		)
														rename_keys.append((F'''transformer.blocks.{i}.norm1.bias''', F'''vilt.encoder.layer.{i}.layernorm_before.bias''')		)
														rename_keys.append(
														    (F'''transformer.blocks.{i}.attn.proj.weight''', F'''vilt.encoder.layer.{i}.attention.output.dense.weight''')		)
														rename_keys.append(
														    (F'''transformer.blocks.{i}.attn.proj.bias''', F'''vilt.encoder.layer.{i}.attention.output.dense.bias''')		)
														rename_keys.append((F'''transformer.blocks.{i}.norm2.weight''', F'''vilt.encoder.layer.{i}.layernorm_after.weight''')		)
														rename_keys.append((F'''transformer.blocks.{i}.norm2.bias''', F'''vilt.encoder.layer.{i}.layernorm_after.bias''')		)
														rename_keys.append(
														    (F'''transformer.blocks.{i}.mlp.fc1.weight''', F'''vilt.encoder.layer.{i}.intermediate.dense.weight''')		)
														rename_keys.append((F'''transformer.blocks.{i}.mlp.fc1.bias''', F'''vilt.encoder.layer.{i}.intermediate.dense.bias''')		)
														rename_keys.append((F'''transformer.blocks.{i}.mlp.fc2.weight''', F'''vilt.encoder.layer.{i}.output.dense.weight''')		)
														rename_keys.append((F'''transformer.blocks.{i}.mlp.fc2.bias''', F'''vilt.encoder.layer.{i}.output.dense.bias''')		)

							# embeddings
							rename_keys.extend(
							    [
							        # text embeddings
							        ("""text_embeddings.word_embeddings.weight""", """vilt.embeddings.text_embeddings.word_embeddings.weight"""),
							        (
							            """text_embeddings.position_embeddings.weight""",
							            """vilt.embeddings.text_embeddings.position_embeddings.weight""",
							        ),
							        ("""text_embeddings.position_ids""", """vilt.embeddings.text_embeddings.position_ids"""),
							        (
							            """text_embeddings.token_type_embeddings.weight""",
							            """vilt.embeddings.text_embeddings.token_type_embeddings.weight""",
							        ),
							        ("""text_embeddings.LayerNorm.weight""", """vilt.embeddings.text_embeddings.LayerNorm.weight"""),
							        ("""text_embeddings.LayerNorm.bias""", """vilt.embeddings.text_embeddings.LayerNorm.bias"""),
							        # patch embeddings
							        ("""transformer.cls_token""", """vilt.embeddings.cls_token"""),
							        ("""transformer.patch_embed.proj.weight""", """vilt.embeddings.patch_embeddings.projection.weight"""),
							        ("""transformer.patch_embed.proj.bias""", """vilt.embeddings.patch_embeddings.projection.bias"""),
							        ("""transformer.pos_embed""", """vilt.embeddings.position_embeddings"""),
							        # token type embeddings
							        ("""token_type_embeddings.weight""", """vilt.embeddings.token_type_embeddings.weight"""),
							    ]		)

							# final layernorm + pooler
							rename_keys.extend(
							    [
							        ("""transformer.norm.weight""", """vilt.layernorm.weight"""),
							        ("""transformer.norm.bias""", """vilt.layernorm.bias"""),
							        ("""pooler.dense.weight""", """vilt.pooler.dense.weight"""),
							        ("""pooler.dense.bias""", """vilt.pooler.dense.bias"""),
							    ]		)

							# classifier head(s)
							if vqa_model:
														# classification head
														rename_keys.extend(
														    [
														        ("""vqa_classifier.0.weight""", """classifier.0.weight"""),
														        ("""vqa_classifier.0.bias""", """classifier.0.bias"""),
														        ("""vqa_classifier.1.weight""", """classifier.1.weight"""),
														        ("""vqa_classifier.1.bias""", """classifier.1.bias"""),
														        ("""vqa_classifier.3.weight""", """classifier.3.weight"""),
														        ("""vqa_classifier.3.bias""", """classifier.3.bias"""),
														    ]		)
							elif nlvr_model:
														# classification head
														rename_keys.extend(
														    [
														        ("""nlvr2_classifier.0.weight""", """classifier.0.weight"""),
														        ("""nlvr2_classifier.0.bias""", """classifier.0.bias"""),
														        ("""nlvr2_classifier.1.weight""", """classifier.1.weight"""),
														        ("""nlvr2_classifier.1.bias""", """classifier.1.bias"""),
														        ("""nlvr2_classifier.3.weight""", """classifier.3.weight"""),
														        ("""nlvr2_classifier.3.bias""", """classifier.3.bias"""),
														    ]		)
							else:
														pass

							return rename_keys



def       __lowerCamelCase  (					_lowercase	,				_lowercase		)      ->			str:
							for i in range(config.num_hidden_layers		):
														UpperCAmelCase     :		int							     =		"""vilt."""
														# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
														UpperCAmelCase     :		List[Any]							     =		state_dict.pop(F'''transformer.blocks.{i}.attn.qkv.weight'''		)
														UpperCAmelCase     :		Any							     =		state_dict.pop(F'''transformer.blocks.{i}.attn.qkv.bias'''		)
														# next, add query, keys and values (in that order) to the state dict
														UpperCAmelCase     :		Any							     =		in_proj_weight[
														    : config.hidden_size, :
														]
														UpperCAmelCase     :		List[str]							     =		in_proj_bias[: config.hidden_size]
														UpperCAmelCase     :		List[str]							     =		in_proj_weight[
														    config.hidden_size : config.hidden_size * 2, :
														]
														UpperCAmelCase     :		Dict							     =		in_proj_bias[
														    config.hidden_size : config.hidden_size * 2
														]
														UpperCAmelCase     :		Union[str, Any]							     =		in_proj_weight[
														    -config.hidden_size :, :
														]
														UpperCAmelCase     :		List[Any]							     =		in_proj_bias[-config.hidden_size :]



def       __lowerCamelCase  (					_lowercase		)      ->			List[str]:
							UpperCAmelCase     :		str							     =		["""head.weight""", """head.bias"""]
							for k in ignore_keys:
														state_dict.pop(_lowercase	,				_lowercase		)



def       __lowerCamelCase  (					_lowercase	,				_lowercase	,				_lowercase		)      ->			List[Any]:
							UpperCAmelCase     :		Optional[int]							     =		dct.pop(_lowercase		)
							UpperCAmelCase     :		str							     =		val



@torch.no_grad()
def       __lowerCamelCase  (					_lowercase	,				_lowercase		)      ->			Dict:
							UpperCAmelCase     :		List[Any]							     =		ViltConfig(image_size=3_8_4	,				patch_size=3_2	,				tie_word_embeddings=_lowercase		)
							UpperCAmelCase     :		str							     =		False
							UpperCAmelCase     :		List[str]							     =		False
							UpperCAmelCase     :		Tuple							     =		False
							UpperCAmelCase     :		Optional[Any]							     =		False
							if "vqa" in checkpoint_url:
														UpperCAmelCase     :		Tuple							     =		True
														UpperCAmelCase     :		Union[str, Any]							     =		3_1_2_9
														UpperCAmelCase     :		Optional[Any]							     =		"""huggingface/label-files"""
														UpperCAmelCase     :		Any							     =		"""vqa2-id2label.json"""
														UpperCAmelCase     :		Union[str, Any]							     =		json.load(open(hf_hub_download(_lowercase	,				_lowercase	,				repo_type="""dataset"""		)	,				"""r"""		)		)
														UpperCAmelCase     :		Union[str, Any]							     =		{int(_lowercase		): v for k, v in idalabel.items()}
														UpperCAmelCase     :		Optional[int]							     =		idalabel
														UpperCAmelCase     :		Any							     =		{v: k for k, v in idalabel.items()}
														UpperCAmelCase     :		Tuple							     =		ViltForQuestionAnswering(_lowercase		)
							elif "nlvr" in checkpoint_url:
														UpperCAmelCase     :		Optional[Any]							     =		True
														UpperCAmelCase     :		Dict							     =		2
														UpperCAmelCase     :		List[Any]							     =		{0: """False""", 1: """True"""}
														UpperCAmelCase     :		str							     =		{v: k for k, v in config.idalabel.items()}
														UpperCAmelCase     :		Union[str, Any]							     =		3
														UpperCAmelCase     :		int							     =		ViltForImagesAndTextClassification(_lowercase		)
							elif "irtr" in checkpoint_url:
														UpperCAmelCase     :		int							     =		True
														UpperCAmelCase     :		Union[str, Any]							     =		ViltForImageAndTextRetrieval(_lowercase		)
							elif "mlm_itm" in checkpoint_url:
														UpperCAmelCase     :		List[str]							     =		True
														UpperCAmelCase     :		List[Any]							     =		ViltForMaskedLM(_lowercase		)
							else:
														raise ValueError("""Unknown model type"""		)

							# load state_dict of original model, remove and rename some keys
							UpperCAmelCase     :		int							     =		torch.hub.load_state_dict_from_url(_lowercase	,				map_location="""cpu"""		)["""state_dict"""]
							UpperCAmelCase     :		Union[str, Any]							     =		create_rename_keys(_lowercase	,				_lowercase	,				_lowercase	,				_lowercase		)
							for src, dest in rename_keys:
														rename_key(_lowercase	,				_lowercase	,				_lowercase		)
							read_in_q_k_v(_lowercase	,				_lowercase		)
							if mlm_model or irtr_model:
														UpperCAmelCase     :		Optional[Any]							     =		["""itm_score.fc.weight""", """itm_score.fc.bias"""]
														for k in ignore_keys:
																					state_dict.pop(_lowercase	,				_lowercase		)

    # load state dict into HuggingFace model
							model.eval()
							if mlm_model:
														UpperCAmelCase     ,     UpperCAmelCase     :		int							     =		model.load_state_dict(_lowercase	,				strict=_lowercase		)
														assert missing_keys == ["mlm_score.decoder.bias"]
							else:
														model.load_state_dict(_lowercase		)

							# Define processor
							UpperCAmelCase     :		Union[str, Any]							     =		ViltImageProcessor(size=3_8_4		)
							UpperCAmelCase     :		Optional[int]							     =		BertTokenizer.from_pretrained("""bert-base-uncased"""		)
							UpperCAmelCase     :		Any							     =		ViltProcessor(_lowercase	,				_lowercase		)

							# Forward pass on example inputs (image + text)
							if nlvr_model:
														UpperCAmelCase     :		Union[str, Any]							     =		Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""	,				stream=_lowercase		).raw		)
														UpperCAmelCase     :		Optional[int]							     =		Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""	,				stream=_lowercase		).raw		)
														UpperCAmelCase     :		str							     =		(
														    """The left image contains twice the number of dogs as the right image, and at least two dogs in total are"""
														    """ standing."""
														)
														UpperCAmelCase     :		str							     =		processor(_lowercase	,				_lowercase	,				return_tensors="""pt"""		)
														UpperCAmelCase     :		str							     =		processor(_lowercase	,				_lowercase	,				return_tensors="""pt"""		)
														UpperCAmelCase     :		int							     =		model(
														    input_ids=encoding_a.input_ids	,				pixel_values=encoding_a.pixel_values	,				pixel_values_a=encoding_a.pixel_values	,				)
							else:
														UpperCAmelCase     :		Dict							     =		Image.open(requests.get("""http://images.cocodataset.org/val2017/000000039769.jpg"""	,				stream=_lowercase		).raw		)
														if mlm_model:
																					UpperCAmelCase     :		Optional[int]							     =		"""a bunch of [MASK] laying on a [MASK]."""
														else:
																					UpperCAmelCase     :		Any							     =		"""How many cats are there?"""
														UpperCAmelCase     :		Optional[int]							     =		processor(_lowercase	,				_lowercase	,				return_tensors="""pt"""		)
														UpperCAmelCase     :		List[str]							     =		model(**_lowercase		)

							# Verify outputs
							if mlm_model:
														UpperCAmelCase     :		Optional[int]							     =		torch.Size([1, 1_1, 3_0_5_2_2]		)
														UpperCAmelCase     :		int							     =		torch.tensor([-12.5061, -12.5123, -12.5174]		)
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]	,				_lowercase	,				atol=1e-4		)

														# verify masked token prediction equals "cats"
														UpperCAmelCase     :		List[Any]							     =		outputs.logits[0, 4, :].argmax(-1		).item()
														assert tokenizer.decode([predicted_id]		) == "cats"
							elif vqa_model:
														UpperCAmelCase     :		Optional[int]							     =		torch.Size([1, 3_1_2_9]		)
														UpperCAmelCase     :		Optional[Any]							     =		torch.tensor([-15.9495, -18.1472, -10.3041]		)
														assert torch.allclose(outputs.logits[0, :3]	,				_lowercase	,				atol=1e-4		)
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]	,				_lowercase	,				atol=1e-4		)

														# verify vqa prediction equals "2"
														UpperCAmelCase     :		int							     =		outputs.logits.argmax(-1		).item()
														assert model.config.idalabel[predicted_idx] == "2"
							elif nlvr_model:
														UpperCAmelCase     :		Optional[Any]							     =		torch.Size([1, 2]		)
														UpperCAmelCase     :		Optional[Any]							     =		torch.tensor([-2.8721, 2.1291]		)
														assert torch.allclose(outputs.logits[0, :3]	,				_lowercase	,				atol=1e-4		)
														assert outputs.logits.shape == expected_shape

							Path(_lowercase		).mkdir(exist_ok=_lowercase		)
							print(F'''Saving model and processor to {pytorch_dump_folder_path}'''		)
							model.save_pretrained(_lowercase		)
							processor.save_pretrained(_lowercase		)


if __name__ == "__main__":
							a   :			str					  =    argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    """--checkpoint_url""",
							    default="""https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt""",
							    type=str,
							    help="""URL of the checkpoint you'd like to convert.""",
							)
							parser.add_argument(
							    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
							)

							a   :			Tuple					  =    parser.parse_args()
							convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)






'''simple docstring'''




from collections import UserDict
from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
							from PIL import Image

							from ..image_utils import load_image

if is_torch_available():
							from ..models.auto.modeling_auto import MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING

if is_tf_available():
							from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
							from ..tf_utils import stable_softmax

a   :			Union[str, Any]					  =    logging.get_logger(__name__)

@add_end_docstrings(__magic_name__			)
class 			UpperCamelCase_ (							__magic_name__			):





	def __init__(  self		,		**A    )			->			List[str]:
								super().__init__(**A    )

								requires_backends(self		,		"""vision"""    )
								self.check_model_type(
								    TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
								    if self.framework == """tf"""
								    else MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING    )





	def __call__(  self		,		A		,		**A    )			->			Optional[Any]:
								return super().__call__(A		,		**A    )





	def  _lowercase(  self		,		**A    )			->			Optional[Any]:
								UpperCAmelCase     :		List[Any]							     =		{}
								if "candidate_labels" in kwargs:
															UpperCAmelCase     :		Dict							     =		kwargs["""candidate_labels"""]
								if "hypothesis_template" in kwargs:
															UpperCAmelCase     :		Optional[Any]							     =		kwargs["""hypothesis_template"""]

								return preprocess_params, {}, {}





	def  _lowercase(  self		,		A		,		A=None		,		A="This is a photo of {}."    )			->			Optional[Any]:
								UpperCAmelCase     :		int							     =		load_image(A    )
								UpperCAmelCase     :		List[str]							     =		self.image_processor(images=[image]		,		return_tensors=self.framework    )
								UpperCAmelCase     :		List[str]							     =		candidate_labels
								UpperCAmelCase     :		Tuple							     =		[hypothesis_template.format(A    ) for x in candidate_labels]
								UpperCAmelCase     :		Union[str, Any]							     =		self.tokenizer(A		,		return_tensors=self.framework		,		padding=A    )
								UpperCAmelCase     :		Union[str, Any]							     =		[text_inputs]
								return inputs





	def  _lowercase(  self		,		A    )			->			Optional[int]:
								UpperCAmelCase     :		List[Any]							     =		model_inputs.pop("""candidate_labels"""    )
								UpperCAmelCase     :		Optional[Any]							     =		model_inputs.pop("""text_inputs"""    )
								if isinstance(text_inputs[0]		,		A    ):
															UpperCAmelCase     :		Optional[Any]							     =		text_inputs[0]
								else:
															# Batching case.
															UpperCAmelCase     :		Any							     =		text_inputs[0][0]

								UpperCAmelCase     :		Dict							     =		self.model(**A		,		**A    )

								UpperCAmelCase     :		List[Any]							     =		{
								    """candidate_labels""": candidate_labels,
								    """logits""": outputs.logits_per_image,
								}
								return model_outputs






	def  _lowercase(  self		,		A    )			->			Union[str, Any]:
								UpperCAmelCase     :		int							     =		model_outputs.pop("""candidate_labels"""    )
								UpperCAmelCase     :		int							     =		model_outputs["""logits"""][0]
								if self.framework == "pt":
															UpperCAmelCase     :		Optional[int]							     =		logits.softmax(dim=-1    ).squeeze(-1    )
															UpperCAmelCase     :		Any							     =		probs.tolist()
															if not isinstance(A		,		A    ):
																						UpperCAmelCase     :		Any							     =		[scores]
								elif self.framework == "tf":
															UpperCAmelCase     :		List[str]							     =		stable_softmax(A		,		axis=-1    )
															UpperCAmelCase     :		Union[str, Any]							     =		probs.numpy().tolist()
								else:
															raise ValueError(f'''Unsupported framework: {self.framework}'''    )

								UpperCAmelCase     :		Any							     =		[
								    {"""score""": score, """label""": candidate_label}
								    for score, candidate_label in sorted(zip(A		,		A    )		,		key=lambda A    : -x[0]    )
								]
								return result

