





"""simple docstring"""
import unittest

from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
from transformers.pipelines import pipeline
from transformers.pipelines.document_question_answering import apply_tesseract
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_detectrona,
    require_pytesseract,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
		from PIL import Image

		from transformers.image_utils import load_image
else:






		class 					_UpperCAmelCase   :


							@staticmethod
							def 	lowerCamelCase       (					*__UpperCamelCase			:Any     ,       **__UpperCamelCase			:Any		):
														pass







		def 					A__				(			UpperCamelCase       ):
									return None


# This is a pinned image from a specific revision of a document question answering space, hosted by HuggingFace,
# so we can expect it to be available.
_snake_case     :		Union[str, Any]       						= (
    'https://huggingface.co/spaces/impira/docquery/resolve/2f6c96314dc84dfda62d40de9da55f2f5165d403/invoice.png'
)






@is_pipeline_test
@require_torch
@require_vision
class 					_UpperCAmelCase   (	unittest.TestCase ):
					UpperCamelCase      =							MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING


					@require_pytesseract
					@require_vision
					def 	lowerCamelCase       (					self			:Any     ,       __UpperCamelCase			:Optional[int]     ,       __UpperCamelCase			:List[Any]     ,       __UpperCamelCase			:int		):
												A		=  pipeline(
												    "document-question-answering"     ,       model=__UpperCamelCase     ,       tokenizer=__UpperCamelCase     ,       image_processor=__UpperCamelCase		)

												A		=  INVOICE_URL
												A		=  list(zip(*apply_tesseract(load_image(__UpperCamelCase		)     ,       __UpperCamelCase     ,       ""		)		)		)
												A		=  "What is the placebo?"
												A		=  [
												    {
												        "image": load_image(__UpperCamelCase		),
												        "question": question,
												    },
												    {
												        "image": image,
												        "question": question,
												    },
												    {
												        "image": image,
												        "question": question,
												        "word_boxes": word_boxes,
												    },
												]
												return dqa_pipeline, examples


					def 	lowerCamelCase       (					self			:Dict     ,       __UpperCamelCase			:Optional[Any]     ,       __UpperCamelCase			:Optional[int]		):
												A		=  dqa_pipeline(__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(
												    __UpperCamelCase     ,       [
												        [
												            {"score": ANY(__UpperCamelCase		), "answer": ANY(__UpperCamelCase		), "start": ANY(__UpperCamelCase		), "end": ANY(__UpperCamelCase		)},
												            {"score": ANY(__UpperCamelCase		), "answer": ANY(__UpperCamelCase		), "start": ANY(__UpperCamelCase		), "end": ANY(__UpperCamelCase		)},
												        ]
												    ]
												    * 3     ,       )


					@require_torch
					@require_detectrona
					@require_pytesseract
					def 	lowerCamelCase       (					self			:List[str]		):
												A		=  pipeline("document-question-answering"     ,       model="hf-internal-testing/tiny-random-layoutlmv2"		)
												A		=  INVOICE_URL
												A		=  "How many cats are there?"

												A		=  [
												    {"score": 0.0_001, "answer": "oy 2312/2019", "start": 38, "end": 39},
												    {"score": 0.0_001, "answer": "oy 2312/2019 DUE", "start": 38, "end": 40},
												]
												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       __UpperCamelCase		)

												A		=  dqa_pipeline({"image": image, "question": question}     ,       top_k=2		)
												self.assertEqual(nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       __UpperCamelCase		)

												# This image does not detect ANY text in it, meaning layoutlmv2 should fail.
												# Empty answer probably
												A		=  "./tests/fixtures/tests_samples/COCO/000000039769.png"
												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(__UpperCamelCase     ,       []		)

												# We can optionnally pass directly the words and bounding boxes
												A		=  "./tests/fixtures/tests_samples/COCO/000000039769.png"
												A		=  []
												A		=  []
												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       words=__UpperCamelCase     ,       boxes=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(__UpperCamelCase     ,       []		)


					@slow
					@require_torch
					@require_detectrona
					@require_pytesseract
					def 	lowerCamelCase       (					self			:Union[str, Any]		):
												A		=  pipeline(
												    "document-question-answering"     ,       model="tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"     ,       revision="9977165"     ,       )
												A		=  INVOICE_URL
												A		=  "What is the invoice number?"

												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.9_944, "answer": "us-001", "start": 16, "end": 16},
												        {"score": 0.0_009, "answer": "us-001", "start": 16, "end": 16},
												    ]     ,       )

												A		=  dqa_pipeline({"image": image, "question": question}     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.9_944, "answer": "us-001", "start": 16, "end": 16},
												        {"score": 0.0_009, "answer": "us-001", "start": 16, "end": 16},
												    ]     ,       )

												A		=  dqa_pipeline(
												    [{"image": image, "question": question}, {"image": image, "question": question}]     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        [
												            {"score": 0.9_944, "answer": "us-001", "start": 16, "end": 16},
												            {"score": 0.0_009, "answer": "us-001", "start": 16, "end": 16},
												        ],
												    ]
												    * 2     ,       )


					@slow
					@require_torch
					@require_detectrona
					@require_pytesseract
					def 	lowerCamelCase       (					self			:Tuple		):
												A		=  pipeline(
												    "document-question-answering"     ,       model="tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"     ,       revision="9977165"     ,       max_seq_len=50     ,       )
												A		=  INVOICE_URL
												A		=  "What is the invoice number?"

												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.9_974, "answer": "1110212019", "start": 23, "end": 23},
												        {"score": 0.9_948, "answer": "us-001", "start": 16, "end": 16},
												    ]     ,       )

												A		=  dqa_pipeline({"image": image, "question": question}     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.9_974, "answer": "1110212019", "start": 23, "end": 23},
												        {"score": 0.9_948, "answer": "us-001", "start": 16, "end": 16},
												    ]     ,       )

												A		=  dqa_pipeline(
												    [{"image": image, "question": question}, {"image": image, "question": question}]     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        [
												            {"score": 0.9_974, "answer": "1110212019", "start": 23, "end": 23},
												            {"score": 0.9_948, "answer": "us-001", "start": 16, "end": 16},
												        ]
												    ]
												    * 2     ,       )


					@slow
					@require_torch
					@require_pytesseract
					@require_vision
					def 	lowerCamelCase       (					self			:Dict		):
												A		=  AutoTokenizer.from_pretrained(
												    "impira/layoutlm-document-qa"     ,       revision="3dc6de3"     ,       add_prefix_space=__UpperCamelCase		)
												A		=  pipeline(
												    "document-question-answering"     ,       model="impira/layoutlm-document-qa"     ,       tokenizer=__UpperCamelCase     ,       revision="3dc6de3"     ,       )
												A		=  INVOICE_URL
												A		=  "What is the invoice number?"

												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.4_251, "answer": "us-001", "start": 16, "end": 16},
												        {"score": 0.0_819, "answer": "1110212019", "start": 23, "end": 23},
												    ]     ,       )

												A		=  dqa_pipeline({"image": image, "question": question}     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.4_251, "answer": "us-001", "start": 16, "end": 16},
												        {"score": 0.0_819, "answer": "1110212019", "start": 23, "end": 23},
												    ]     ,       )

												A		=  dqa_pipeline(
												    [{"image": image, "question": question}, {"image": image, "question": question}]     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        [
												            {"score": 0.4_251, "answer": "us-001", "start": 16, "end": 16},
												            {"score": 0.0_819, "answer": "1110212019", "start": 23, "end": 23},
												        ]
												    ]
												    * 2     ,       )

												A		=  list(zip(*apply_tesseract(load_image(__UpperCamelCase		)     ,       __UpperCamelCase     ,       ""		)		)		)

												# This model should also work if `image` is set to None
												A		=  dqa_pipeline({"image": None, "word_boxes": word_boxes, "question": question}     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.4_251, "answer": "us-001", "start": 16, "end": 16},
												        {"score": 0.0_819, "answer": "1110212019", "start": 23, "end": 23},
												    ]     ,       )


					@slow
					@require_torch
					@require_pytesseract
					@require_vision
					def 	lowerCamelCase       (					self			:int		):
												A		=  AutoTokenizer.from_pretrained(
												    "impira/layoutlm-document-qa"     ,       revision="3dc6de3"     ,       add_prefix_space=__UpperCamelCase		)
												A		=  pipeline(
												    "document-question-answering"     ,       model="impira/layoutlm-document-qa"     ,       tokenizer=__UpperCamelCase     ,       revision="3dc6de3"     ,       max_seq_len=50     ,       )
												A		=  INVOICE_URL
												A		=  "What is the invoice number?"

												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.9_999, "answer": "us-001", "start": 16, "end": 16},
												        {"score": 0.9_998, "answer": "us-001", "start": 16, "end": 16},
												    ]     ,       )

												A		=  dqa_pipeline(
												    [{"image": image, "question": question}, {"image": image, "question": question}]     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        [
												            {"score": 0.9_999, "answer": "us-001", "start": 16, "end": 16},
												            {"score": 0.9_998, "answer": "us-001", "start": 16, "end": 16},
												        ]
												    ]
												    * 2     ,       )

												A		=  list(zip(*apply_tesseract(load_image(__UpperCamelCase		)     ,       __UpperCamelCase     ,       ""		)		)		)

												# This model should also work if `image` is set to None
												A		=  dqa_pipeline({"image": None, "word_boxes": word_boxes, "question": question}     ,       top_k=2		)
												self.assertEqual(
												    nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [
												        {"score": 0.9_999, "answer": "us-001", "start": 16, "end": 16},
												        {"score": 0.9_998, "answer": "us-001", "start": 16, "end": 16},
												    ]     ,       )


					@slow
					@require_torch
					def 	lowerCamelCase       (					self			:Optional[Any]		):
												A		=  pipeline(
												    "document-question-answering"     ,       model="naver-clova-ix/donut-base-finetuned-docvqa"     ,       tokenizer=AutoTokenizer.from_pretrained("naver-clova-ix/donut-base-finetuned-docvqa"		)     ,       feature_extractor="naver-clova-ix/donut-base-finetuned-docvqa"     ,       )

												A		=  INVOICE_URL
												A		=  "What is the invoice number?"
												A		=  dqa_pipeline(image=__UpperCamelCase     ,       question=__UpperCamelCase     ,       top_k=2		)
												self.assertEqual(nested_simplify(__UpperCamelCase     ,       decimals=4		)     ,       [{"answer": "us-001"}]		)


					@require_tf
					@unittest.skip("Document question answering not implemented in TF"		)
					def 	lowerCamelCase       (					self			:Tuple		):
												pass









"""simple docstring"""
import argparse

import torch
from torch import nn

from transformers import SpeechaTextConfig, SpeechaTextForConditionalGeneration






def 					A__				(			UpperCamelCase       ):
							A		=  [
							    "encoder.version",
							    "decoder.version",
							    "model.encoder.version",
							    "model.decoder.version",
							    "decoder.output_projection.weight",
							    "_float_tensor",
							    "encoder.embed_positions._float_tensor",
							    "decoder.embed_positions._float_tensor",
							]
							for k in ignore_keys:
														state_dict.pop(UpperCamelCase					,  UpperCamelCase       )






def 					A__				(			UpperCamelCase       ):
							A		=  list(s_dict.keys()       )
							for key in keys:
														if "transformer_layers" in key:
																					A		=  s_dict.pop(UpperCamelCase       )
														elif "subsample" in key:
																					A		=  s_dict.pop(UpperCamelCase       )






def 					A__				(			UpperCamelCase       ):
							A,	A		=  emb.weight.shape
							A		=  nn.Linear(UpperCamelCase					,  UpperCamelCase					,  bias=UpperCamelCase       )
							A		=  emb.weight.data
							return lin_layer






def 					A__				(			UpperCamelCase					,  UpperCamelCase       ):
							A		=  torch.load(UpperCamelCase					,  map_location="cpu"       )
							A		=  mam_aaa["args"]
							A		=  mam_aaa["model"]
							A		=  state_dict["decoder.output_projection.weight"]

							remove_ignore_keys_(UpperCamelCase       )
							rename_keys(UpperCamelCase       )

							A		=  state_dict["decoder.embed_tokens.weight"].shape[0]

							A		=  args.share_decoder_input_output_embed

							A		=  [int(UpperCamelCase       ) for i in args.conv_kernel_sizes.split(","       )]
							A		=  SpeechaTextConfig(
							    vocab_size=UpperCamelCase					,  max_source_positions=args.max_source_positions					,  max_target_positions=args.max_target_positions					,  encoder_layers=args.encoder_layers					,  decoder_layers=args.decoder_layers					,  encoder_attention_heads=args.encoder_attention_heads					,  decoder_attention_heads=args.decoder_attention_heads					,  encoder_ffn_dim=args.encoder_ffn_embed_dim					,  decoder_ffn_dim=args.decoder_ffn_embed_dim					,  d_model=args.encoder_embed_dim					,  dropout=args.dropout					,  attention_dropout=args.attention_dropout					,  activation_dropout=args.activation_dropout					,  activation_function="relu"					,  num_conv_layers=len(UpperCamelCase       )					,  conv_channels=args.conv_channels					,  conv_kernel_sizes=UpperCamelCase					,  input_feat_per_channel=args.input_feat_per_channel					,  input_channels=args.input_channels					,  tie_word_embeddings=UpperCamelCase					,  num_beams=5					,  max_length=200					,  use_cache=UpperCamelCase					,  decoder_start_token_id=2					,  early_stopping=UpperCamelCase					,  )

							A		=  SpeechaTextForConditionalGeneration(UpperCamelCase       )
							A,	A		=  model.model.load_state_dict(UpperCamelCase					,  strict=UpperCamelCase       )
							if len(UpperCamelCase       ) > 0 and not set(UpperCamelCase       ) <= {
							    "encoder.embed_positions.weights",
							    "decoder.embed_positions.weights",
							}:
														raise ValueError(
														    "Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,"
														    F" but all the following weights are missing {missing}"       )

							if tie_embeds:
														A		=  make_linear_from_emb(model.model.decoder.embed_tokens       )
							else:
														A		=  lm_head_weights

							model.save_pretrained(UpperCamelCase       )


if __name__ == "__main__":
		_snake_case     :		Optional[int]       						= argparse.ArgumentParser()
		# Required parameters
		parser.add_argument('--fairseq_path', type=str, help='Path to the fairseq model (.pt) file.')
		parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
		_snake_case     :		str       						= parser.parse_args()
		convert_fairseq_sat_checkpoint_to_tfms(args.fairseq_path, args.pytorch_dump_folder_path)


