





"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import numpy
		import tensorflow as tf

		from transformers import (
		    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    BertConfig,
		    DPRConfig,
		    TFDPRContextEncoder,
		    TFDPRQuestionEncoder,
		    TFDPRReader,
		)



class 					__UpperCAmelCase							:
				def __init__(		self    :			List[str]	, a_    :			Optional[Any]	, a_    :			int=13	, a_    :			Any=7	, a_    :			int=True	, a_    :			Optional[Any]=True	, a_    :			Optional[int]=True	, a_    :			Tuple=True	, a_    :			int=99	, a_    :			Optional[int]=32	, a_    :			str=2	, a_    :			Optional[Any]=4	, a_    :			int=37	, a_    :			List[str]="gelu"	, a_    :			str=0.1	, a_    :			str=0.1	, a_    :			Dict=5_12	, a_    :			Union[str, Any]=16	, a_    :			Dict=2	, a_    :			Dict=0.02	, a_    :			Optional[Any]=3	, a_    :			Tuple=4	, a_    :			Optional[int]=None	, a_    :			int=0	, )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	Tuple		 =			parent
							a__  :	int		 =			batch_size
							a__  :	int		 =			seq_length
							a__  :	Tuple		 =			is_training
							a__  :	Optional[int]		 =			use_input_mask
							a__  :	str		 =			use_token_type_ids
							a__  :	Tuple		 =			use_labels
							a__  :	Any		 =			vocab_size
							a__  :	List[str]		 =			hidden_size
							a__  :	int		 =			num_hidden_layers
							a__  :	Tuple		 =			num_attention_heads
							a__  :	List[Any]		 =			intermediate_size
							a__  :	Optional[int]		 =			hidden_act
							a__  :	Union[str, Any]		 =			hidden_dropout_prob
							a__  :	Dict		 =			attention_probs_dropout_prob
							a__  :	int		 =			max_position_embeddings
							a__  :	Optional[int]		 =			type_vocab_size
							a__  :	List[str]		 =			type_sequence_label_size
							a__  :	Union[str, Any]		 =			initializer_range
							a__  :	Tuple		 =			num_labels
							a__  :	Tuple		 =			num_choices
							a__  :	Union[str, Any]		 =			scope
							a__  :	List[str]		 =			projection_dim
				def       UpperCAmelCase      (		self    :			int    )	->	str:



							'''simple docstring'''



							a__  :	Tuple		 =			ids_tensor([self.batch_size, self.seq_length]	, self.vocab_size    )

							a__  :	List[Any]		 =			None
							if self.use_input_mask:
										# follow test_modeling_tf_ctrl.py
										a__  :	List[Any]		 =			random_attention_mask([self.batch_size, self.seq_length]    )

							a__  :	List[Any]		 =			None
							if self.use_token_type_ids:
										a__  :	Union[str, Any]		 =			ids_tensor([self.batch_size, self.seq_length]	, self.type_vocab_size    )

							a__  :	Dict		 =			None
							a__  :	Optional[int]		 =			None
							a__  :	str		 =			None
							if self.use_labels:
										a__  :	Optional[int]		 =			ids_tensor([self.batch_size]	, self.type_sequence_label_size    )
										a__  :	int		 =			ids_tensor([self.batch_size, self.seq_length]	, self.num_labels    )
										a__  :	Optional[int]		 =			ids_tensor([self.batch_size]	, self.num_choices    )

							a__  :	Any		 =			BertConfig(
							    vocab_size=self.vocab_size	, hidden_size=self.hidden_size	, num_hidden_layers=self.num_hidden_layers	, num_attention_heads=self.num_attention_heads	, intermediate_size=self.intermediate_size	, hidden_act=self.hidden_act	, hidden_dropout_prob=self.hidden_dropout_prob	, attention_probs_dropout_prob=self.attention_probs_dropout_prob	, max_position_embeddings=self.max_position_embeddings	, type_vocab_size=self.type_vocab_size	, is_decoder=a_	, initializer_range=self.initializer_range	, )
							a__  :	str		 =			DPRConfig(projection_dim=self.projection_dim	, **config.to_dict()    )

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
				def       UpperCAmelCase      (		self    :			Dict	, a_    :			Dict	, a_    :			List[Any]	, a_    :			int	, a_    :			List[str]	, a_    :			List[Any]	, a_    :			Tuple	, a_    :			Union[str, Any]    )	->	Dict:



							'''simple docstring'''



							a__  :	int		 =			TFDPRContextEncoder(config=a_    )
							a__  :	Union[str, Any]		 =			model(a_	, attention_mask=a_	, token_type_ids=a_    )
							a__  :	Any		 =			model(a_	, token_type_ids=a_    )
							a__  :	int		 =			model(a_    )
							self.parent.assertEqual(result.pooler_output.shape	, (self.batch_size, self.projection_dim or self.hidden_size)    )
				def       UpperCAmelCase      (		self    :			Dict	, a_    :			Union[str, Any]	, a_    :			Optional[int]	, a_    :			Any	, a_    :			List[Any]	, a_    :			Tuple	, a_    :			Any	, a_    :			Dict    )	->	Union[str, Any]:



							'''simple docstring'''



							a__  :	Tuple		 =			TFDPRQuestionEncoder(config=a_    )
							a__  :	List[Any]		 =			model(a_	, attention_mask=a_	, token_type_ids=a_    )
							a__  :	Dict		 =			model(a_	, token_type_ids=a_    )
							a__  :	List[Any]		 =			model(a_    )
							self.parent.assertEqual(result.pooler_output.shape	, (self.batch_size, self.projection_dim or self.hidden_size)    )
				def       UpperCAmelCase      (		self    :			List[Any]	, a_    :			Union[str, Any]	, a_    :			Dict	, a_    :			Optional[int]	, a_    :			str	, a_    :			List[Any]	, a_    :			List[str]	, a_    :			List[Any]    )	->	str:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			TFDPRReader(config=a_    )
							a__  :	str		 =			model(a_	, attention_mask=a_    )

							self.parent.assertEqual(result.start_logits.shape	, (self.batch_size, self.seq_length)    )
							self.parent.assertEqual(result.end_logits.shape	, (self.batch_size, self.seq_length)    )
							self.parent.assertEqual(result.relevance_logits.shape	, (self.batch_size,)    )






				def       UpperCAmelCase      (		self    :			Dict    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	str		 =			self.prepare_config_and_inputs()
							(
							    (
							    a__
							) ,	(
							    a__
							) ,	(
							    a__
							) ,	(
							    a__
							) ,	(
							    a__
							) ,	(
							    a__
							) ,	(
							    a__
							) ,	
							)  :	Any		 =			config_and_inputs
							a__  :	Union[str, Any]		 =			{"input_ids": input_ids}
							return config, inputs_dict



@require_tf
class 					__UpperCAmelCase							(						_UpperCamelCase						,  _UpperCamelCase						,  unittest.TestCase  ):
				__lowerCamelCase				:	Optional[Any]				     =				(
				    (
				        TFDPRContextEncoder,
				        TFDPRQuestionEncoder,
				        TFDPRReader,
				    )
				    if is_tf_available()
				    else ()
				)
				__lowerCamelCase				:	Dict				     =				{"feature-extraction": TFDPRQuestionEncoder} if is_tf_available() else {}

				__lowerCamelCase				:	Tuple				     =				False
				__lowerCamelCase				:	Optional[int]				     =				False
				__lowerCamelCase				:	List[str]				     =				False
				__lowerCamelCase				:	Union[str, Any]				     =				False
				__lowerCamelCase				:	List[str]				     =				False
				def       UpperCAmelCase      (		self    :			Tuple    )	->	Union[str, Any]:



							'''simple docstring'''



							a__  :	List[Any]		 =			TFDPRModelTester(self    )
							a__  :	Dict		 =			ConfigTester(self	, config_class=a_	, hidden_size=37    )
				def       UpperCAmelCase      (		self    :			List[str]    )	->	Dict:



							'''simple docstring'''



							self.config_tester.run_common_tests()
				def       UpperCAmelCase      (		self    :			Tuple    )	->	Dict:



							'''simple docstring'''



							a__  :	Optional[int]		 =			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_context_encoder(*a_    )
				def       UpperCAmelCase      (		self    :			Dict    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	Dict		 =			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_question_encoder(*a_    )
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	Any		 =			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_reader(*a_    )






				@slow
				def       UpperCAmelCase      (		self    :			int    )	->	Optional[int]:



							'''simple docstring'''



							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										a__  :	Dict		 =			TFDPRContextEncoder.from_pretrained(a_    )
										self.assertIsNotNone(a_    )

							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										a__  :	List[Any]		 =			TFDPRContextEncoder.from_pretrained(a_    )
										self.assertIsNotNone(a_    )

							for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										a__  :	Any		 =			TFDPRQuestionEncoder.from_pretrained(a_    )
										self.assertIsNotNone(a_    )

							for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										a__  :	Tuple		 =			TFDPRReader.from_pretrained(a_    )
										self.assertIsNotNone(a_    )



@require_tf
class 					__UpperCAmelCase							(						unittest.TestCase  ):
				@slow
				def       UpperCAmelCase      (		self    :			Dict    )	->	Dict:



							'''simple docstring'''



							a__  :	List[str]		 =			TFDPRQuestionEncoder.from_pretrained("facebook/dpr-question_encoder-single-nq-base"    )

							a__  :	int		 =			tf.constant(
							    [[1_01, 75_92, 10_10, 20_03, 20_26, 38_99, 1_01_40, 10_29, 1_02]]    )  # [CLS] hello, is my dog cute? [SEP]
							a__  :	Tuple		 =			model(a_    )[0]  # embedding shape = (1, 768)
							# compare the actual values for a slice.
							a__  :	Optional[int]		 =			tf.constant(
							    [
							        [
							            0.0323_6253,
							            0.1275_3335,
							            0.1681_8509,
							            0.0027_9786,
							            0.389_6933,
							            0.2426_4945,
							            0.217_8971,
							            -0.0233_5227,
							            -0.0848_1959,
							            -0.1432_4117,
							        ]
							    ]    )
							self.assertTrue(numpy.allclose(output[:, :10].numpy()	, expected_slice.numpy()	, atol=1E-4    )    )






"""simple docstring"""






import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_torch, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_torch_available, is_vision_available


if is_torch_available():
		import torch


if is_vision_available():
		from PIL import Image

		from transformers import MgpstrProcessor, ViTImageProcessor



@require_torch
@require_vision
class 					__UpperCAmelCase							(						unittest.TestCase  ):
				__lowerCamelCase				:	int				     =				ViTImageProcessor if is_vision_available() else None
				@property
				def       UpperCAmelCase      (		self    :			List[str]    )	->	List[str]:



							'''simple docstring'''



							return self.image_processor_tester.prepare_image_processor_dict()
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			(3, 32, 1_28)
							a__  :	Any		 =			tempfile.mkdtemp()

							# fmt: off
							a__  :	str		 =			["[GO]", "[s]", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
							# fmt: on
							a__  :	List[Any]		 =			dict(zip(a_	, range(len(a_    )    )    )    )

							a__  :	int		 =			os.path.join(self.tmpdirname	, VOCAB_FILES_NAMES["vocab_file"]    )
							with open(self.vocab_file	, "w"	, encoding="utf-8"    ) as fp:
										fp.write(json.dumps(a_    ) + "\n"    )

							a__  :	Any		 =			{
							    "do_normalize": False,
							    "do_resize": True,
							    "image_processor_type": "ViTImageProcessor",
							    "resample": 3,
							    "size": {"height": 32, "width": 1_28},
							}
							a__  :	Tuple		 =			os.path.join(self.tmpdirname	, a_    )
							with open(self.image_processor_file	, "w"	, encoding="utf-8"    ) as fp:
										json.dump(a_	, a_    )
				def       UpperCAmelCase      (		self    :			List[str]	, **a_    :			List[str]    )	->	Union[str, Any]:



							'''simple docstring'''



							return MgpstrTokenizer.from_pretrained(self.tmpdirname	, **a_    )
				def       UpperCAmelCase      (		self    :			List[Any]	, **a_    :			Union[str, Any]    )	->	Dict:



							'''simple docstring'''



							return ViTImageProcessor.from_pretrained(self.tmpdirname	, **a_    )
				def       UpperCAmelCase      (		self    :			Dict    )	->	int:



							'''simple docstring'''



							shutil.rmtree(self.tmpdirname    )
				def       UpperCAmelCase      (		self    :			List[str]    )	->	Any:



							'''simple docstring'''



							a__  :	Any		 =			np.random.randint(2_55	, size=(3, 30, 4_00)	, dtype=np.uinta    )

							a__  :	List[str]		 =			Image.fromarray(np.moveaxis(a_	, 0	, -1    )    )

							return image_input
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	List[Any]		 =			self.get_tokenizer()
							a__  :	Any		 =			self.get_image_processor()

							a__  :	Optional[int]		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )
							processor.save_pretrained(self.tmpdirname    )
							a__  :	Optional[Any]		 =			MgpstrProcessor.from_pretrained(self.tmpdirname	, use_fast=a_    )

							self.assertEqual(processor.char_tokenizer.get_vocab()	, tokenizer.get_vocab()    )
							self.assertIsInstance(processor.char_tokenizer	, a_    )

							self.assertEqual(processor.image_processor.to_json_string()	, image_processor.to_json_string()    )
							self.assertIsInstance(processor.image_processor	, a_    )
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	str:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			self.get_tokenizer()
							a__  :	Tuple		 =			self.get_image_processor()

							a__  :	Optional[int]		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )
							processor.save_pretrained(self.tmpdirname    )

							a__  :	Union[str, Any]		 =			self.get_tokenizer(bos_token="(BOS)"	, eos_token="(EOS)"    )
							a__  :	Any		 =			self.get_image_processor(do_normalize=a_	, padding_value=1.0    )

							a__  :	Dict		 =			MgpstrProcessor.from_pretrained(
							    self.tmpdirname	, bos_token="(BOS)"	, eos_token="(EOS)"	, do_normalize=a_	, padding_value=1.0    )

							self.assertEqual(processor.char_tokenizer.get_vocab()	, tokenizer_add_kwargs.get_vocab()    )
							self.assertIsInstance(processor.char_tokenizer	, a_    )

							self.assertEqual(processor.image_processor.to_json_string()	, image_processor_add_kwargs.to_json_string()    )
							self.assertIsInstance(processor.image_processor	, a_    )
				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	Dict:



							'''simple docstring'''



							a__  :	str		 =			self.get_image_processor()
							a__  :	Union[str, Any]		 =			self.get_tokenizer()

							a__  :	Any		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )

							a__  :	List[str]		 =			self.prepare_image_inputs()

							a__  :	List[Any]		 =			image_processor(a_	, return_tensors="np"    )
							a__  :	Optional[Any]		 =			processor(images=a_	, return_tensors="np"    )

							for key in input_image_proc.keys():
										self.assertAlmostEqual(input_image_proc[key].sum()	, input_processor[key].sum()	, delta=1E-2    )
				def       UpperCAmelCase      (		self    :			Tuple    )	->	int:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			self.get_image_processor()
							a__  :	List[Any]		 =			self.get_tokenizer()

							a__  :	int		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )

							a__  :	List[str]		 =			"test"

							a__  :	Any		 =			processor(text=a_    )

							a__  :	Tuple		 =			tokenizer(a_    )
							for key in encoded_tok.keys():
										self.assertListEqual(encoded_tok[key]	, encoded_processor[key]    )
				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	Tuple		 =			self.get_image_processor()
							a__  :	List[str]		 =			self.get_tokenizer()

							a__  :	List[Any]		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )

							a__  :	str		 =			"test"
							a__  :	str		 =			self.prepare_image_inputs()

							a__  :	Any		 =			processor(text=a_	, images=a_    )

							self.assertListEqual(list(inputs.keys()    )	, ["pixel_values", "labels"]    )

							# test if it raises when no input is passed
							with pytest.raises(a_    ):
										processor()
				def       UpperCAmelCase      (		self    :			List[str]    )	->	int:



							'''simple docstring'''



							a__  :	Any		 =			self.get_image_processor()
							a__  :	Tuple		 =			self.get_tokenizer()

							a__  :	Dict		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )

							a__  :	Optional[Any]		 =			[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9], [3, 4, 3, 1, 1, 8, 9]]

							a__  :	Any		 =			processor.char_decode(a_    )
							a__  :	str		 =			tokenizer.batch_decode(a_    )
							a__  :	Union[str, Any]		 =			[seq.replace(" "	, ""    ) for seq in decoded_tok]

							self.assertListEqual(a_	, a_    )
				def       UpperCAmelCase      (		self    :			List[Any]    )	->	Dict:



							'''simple docstring'''



							a__  :	List[str]		 =			self.get_image_processor()
							a__  :	Any		 =			self.get_tokenizer()

							a__  :	Any		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )

							a__  :	str		 =			None
							a__  :	Optional[Any]		 =			self.prepare_image_inputs()

							a__  :	Optional[int]		 =			processor(text=a_	, images=a_    )

							self.assertListEqual(list(inputs.keys()    )	, processor.model_input_names    )






				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	List[Any]:



							'''simple docstring'''



							a__  :	List[Any]		 =			self.get_image_processor()
							a__  :	List[str]		 =			self.get_tokenizer()

							a__  :	Optional[Any]		 =			MgpstrProcessor(tokenizer=a_	, image_processor=a_    )

							a__  :	List[str]		 =			torch.randn(1	, 27	, 38    )
							a__  :	Tuple		 =			torch.randn(1	, 27	, 5_02_57    )
							a__  :	List[Any]		 =			torch.randn(1	, 27	, 3_05_22    )

							a__  :	Optional[Any]		 =			processor.batch_decode([char_input, bpe_input, wp_input]    )

							self.assertListEqual(list(results.keys()    )	, ["generated_text", "scores", "char_preds", "bpe_preds", "wp_preds"]    )