




"""simple docstring"""


import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_A													=			get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
  from transformers.models.mbart.modeling_mbart import shift_tokens_right

_A													=			2_5_0_0_0_4
_A													=			2_5_0_0_2_0


@require_sentencepiece
@require_tokenizers
class _lowerCamelCase				(     __snake_case   ,       unittest.TestCase						):
   _lowerCamelCase				:Optional[int]	      =	MBartTokenizer
   _lowerCamelCase				:Optional[Any]	      =	MBartTokenizerFast
   _lowerCamelCase				:Optional[Any]	      =	True
   _lowerCamelCase				:Optional[int]	      =	True






   def        _lowerCAmelCase    (		self   :		str    ) ->						Any:

         """simple docstring"""
         super().setUp()

         # We have a SentencePiece fixture for testing
         lowerCAmelCase__			: Union[str, Any]  					=       MBartTokenizer(a_    ,	keep_accents=a_    )
         tokenizer.save_pretrained(self.tmpdirname    )






   def        _lowerCAmelCase    (		self   :		Union[str, Any]    ) ->						List[Any]:

         """simple docstring"""
         lowerCAmelCase__			: int  					=       MBartTokenizer(a_    ,	keep_accents=a_    )

         lowerCAmelCase__			: Optional[int]  					=       tokenizer.tokenize("""This is a test"""    )
         self.assertListEqual(a_    ,	["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(a_    )    ,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]    ,	)

         lowerCAmelCase__			: Optional[int]  					=       tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
         self.assertListEqual(
             a_    ,	[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ]    ,	)
         lowerCAmelCase__			: List[Any]  					=       tokenizer.convert_tokens_to_ids(a_    )
         self.assertListEqual(
             a_    ,	[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
             ]    ,	)

         lowerCAmelCase__			: Optional[Any]  					=       tokenizer.convert_ids_to_tokens(a_    )
         self.assertListEqual(
             a_    ,	[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """<unk>""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """<unk>""",
                 """.""",
             ]    ,	)






   def        _lowerCAmelCase    (		self   :		Optional[int]    ) ->						Any:

         """simple docstring"""

         if not self.test_slow_tokenizer:
               # as we don't have a slow version, we can't compare the outputs between slow and fast versions
               return

         lowerCAmelCase__			: List[Any]  					=       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart''', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
                     lowerCAmelCase__			: Tuple  					=       self.rust_tokenizer_class.from_pretrained(a_    ,	**a_    )
                     lowerCAmelCase__			: List[str]  					=       self.tokenizer_class.from_pretrained(a_    ,	**a_    )

                     lowerCAmelCase__			: Any  					=       tempfile.mkdtemp()

                     lowerCAmelCase__			: List[Any]  					=       tokenizer_r.save_pretrained(a_    )
                     lowerCAmelCase__			: Union[str, Any]  					=       tokenizer_p.save_pretrained(a_    )

                     # Checks it save with the same files + the tokenizer.json file for the fast one
                     self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )
                     lowerCAmelCase__			: int  					=       tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f    )
                     self.assertSequenceEqual(a_    ,	a_    )

                     # Checks everything loads correctly in the same way
                     lowerCAmelCase__			: int  					=       tokenizer_r.from_pretrained(a_    )
                     lowerCAmelCase__			: Optional[Any]  					=       tokenizer_p.from_pretrained(a_    )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(a_    ,	a_    )    )
                           # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                           # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                     shutil.rmtree(a_    )

                     # Save tokenizer rust, legacy_format=True
                     lowerCAmelCase__			: Any  					=       tempfile.mkdtemp()

                     lowerCAmelCase__			: Tuple  					=       tokenizer_r.save_pretrained(a_    ,	legacy_format=a_    )
                     lowerCAmelCase__			: Union[str, Any]  					=       tokenizer_p.save_pretrained(a_    )

                     # Checks it save with the same files
                     self.assertSequenceEqual(a_    ,	a_    )

                     # Checks everything loads correctly in the same way
                     lowerCAmelCase__			: Any  					=       tokenizer_r.from_pretrained(a_    )
                     lowerCAmelCase__			: Any  					=       tokenizer_p.from_pretrained(a_    )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(a_    ,	a_    )    )

                     shutil.rmtree(a_    )

                     # Save tokenizer rust, legacy_format=False
                     lowerCAmelCase__			: Union[str, Any]  					=       tempfile.mkdtemp()

                     lowerCAmelCase__			: str  					=       tokenizer_r.save_pretrained(a_    ,	legacy_format=a_    )
                     lowerCAmelCase__			: Dict  					=       tokenizer_p.save_pretrained(a_    )

                     # Checks it saved the tokenizer.json file
                     self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )

                     # Checks everything loads correctly in the same way
                     lowerCAmelCase__			: List[str]  					=       tokenizer_r.from_pretrained(a_    )
                     lowerCAmelCase__			: str  					=       tokenizer_p.from_pretrained(a_    )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(a_    ,	a_    )    )

                     shutil.rmtree(a_    )






@require_torch
@require_sentencepiece
@require_tokenizers
class _lowerCamelCase				(     unittest.TestCase						):
   _lowerCamelCase				:Any	      =	"facebook/mbart-large-en-ro"
   _lowerCamelCase				:Optional[Any]	      =	[
       " UN Chief Says There Is No Military Solution in Syria",
       " Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
   ]
   _lowerCamelCase				:Optional[Any]	      =	[
       "Şeful ONU declară că nu există o soluţie militară în Siria",
       "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
       " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
       " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
   ]
   _lowerCamelCase				:str	      =	[8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2, EN_CODE]






   @classmethod
   def        _lowerCAmelCase    (		cls   :		Optional[Any]    ) ->						List[str]:

         """simple docstring"""
         lowerCAmelCase__			: MBartTokenizer  					=       MBartTokenizer.from_pretrained(
             cls.checkpoint_name    ,	src_lang="""en_XX"""    ,	tgt_lang="""ro_RO"""    )
         lowerCAmelCase__			: int  					=       1
         return cls






   def        _lowerCAmelCase    (		self   :		Optional[int]    ) ->						List[str]:

         """simple docstring"""
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]    ,	25_00_01    )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]    ,	25_00_04    )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]    ,	25_00_20    )






   def        _lowerCAmelCase    (		self   :		Optional[int]    ) ->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__			: int  					=       self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens    ,	a_    )






   def        _lowerCAmelCase    (		self   :		Union[str, Any]    ) ->						str:

         """simple docstring"""
         self.assertIn(a_    ,	self.tokenizer.all_special_ids    )
         lowerCAmelCase__			: List[str]  					=       [RO_CODE, 8_84, 90_19, 96, 9, 9_16, 8_67_92, 36, 1_87_43, 1_55_96, 5, 2]
         lowerCAmelCase__			: Any  					=       self.tokenizer.decode(a_    ,	skip_special_tokens=a_    )
         lowerCAmelCase__			: Optional[int]  					=       self.tokenizer.decode(generated_ids[1:]    ,	skip_special_tokens=a_    )
         self.assertEqual(a_    ,	a_    )
         self.assertNotIn(self.tokenizer.eos_token    ,	a_    )






   def        _lowerCAmelCase    (		self   :		Union[str, Any]    ) ->						List[str]:

         """simple docstring"""
         lowerCAmelCase__			: Optional[int]  					=       ['''this is gunna be a long sentence ''' * 20]
         assert isinstance(src_text[0]    ,	a_    )
         lowerCAmelCase__			: Tuple  					=       10
         lowerCAmelCase__			: Optional[Any]  					=       self.tokenizer(a_    ,	max_length=a_    ,	truncation=a_    ).input_ids[0]
         self.assertEqual(ids[-2]    ,	2    )
         self.assertEqual(ids[-1]    ,	a_    )
         self.assertEqual(len(a_    )    ,	a_    )






   def        _lowerCAmelCase    (		self   :		Union[str, Any]    ) ->						List[str]:

         """simple docstring"""
         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]    )    ,	[25_00_26, 25_00_01]    )






   def        _lowerCAmelCase    (		self   :		Optional[int]    ) ->						str:

         """simple docstring"""
         lowerCAmelCase__			: Tuple  					=       tempfile.mkdtemp()
         lowerCAmelCase__			: List[str]  					=       self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(a_    )
         lowerCAmelCase__			: List[Any]  					=       MBartTokenizer.from_pretrained(a_    )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,	a_    )






   @require_torch
   def        _lowerCAmelCase    (		self   :		List[Any]    ) ->						Any:

         """simple docstring"""
         lowerCAmelCase__			: Optional[Any]  					=       self.tokenizer(self.src_text    ,	text_target=self.tgt_text    ,	padding=a_    ,	return_tensors="""pt"""    )
         lowerCAmelCase__			: Union[str, Any]  					=       shift_tokens_right(batch["""labels"""]    ,	self.tokenizer.pad_token_id    )

         # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
         assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
         assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
         assert batch.decoder_input_ids[1][-1] == 2
         assert batch.labels[1][-2:].tolist() == [2, RO_CODE]






   @require_torch
   def        _lowerCAmelCase    (		self   :		List[str]    ) ->						List[Any]:

         """simple docstring"""
         lowerCAmelCase__			: Dict  					=       self.tokenizer(
             self.src_text    ,	text_target=self.tgt_text    ,	padding=a_    ,	truncation=a_    ,	max_length=len(self.expected_src_tokens    )    ,	return_tensors="""pt"""    ,	)

         lowerCAmelCase__			: int  					=       shift_tokens_right(batch["""labels"""]    ,	self.tokenizer.pad_token_id    )

         self.assertIsInstance(a_    ,	a_    )

         self.assertEqual((2, 14)    ,	batch.input_ids.shape    )
         self.assertEqual((2, 14)    ,	batch.attention_mask.shape    )
         lowerCAmelCase__			: Tuple  					=       batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens    ,	a_    )
         self.assertEqual(2    ,	batch.decoder_input_ids[0, -1]    )  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens    ,	[]    )
         self.assertEqual(self.tokenizer.suffix_tokens    ,	[self.tokenizer.eos_token_id, EN_CODE]    )






   def        _lowerCAmelCase    (		self   :		str    ) ->						Optional[int]:

         """simple docstring"""
         lowerCAmelCase__			: int  					=       self.tokenizer(self.src_text    ,	padding=a_    ,	truncation=a_    ,	max_length=3    ,	return_tensors="""pt"""    )
         lowerCAmelCase__			: int  					=       self.tokenizer(
             text_target=self.tgt_text    ,	padding=a_    ,	truncation=a_    ,	max_length=10    ,	return_tensors="""pt"""    )
         lowerCAmelCase__			: Any  					=       targets['''input_ids''']
         lowerCAmelCase__			: List[str]  					=       shift_tokens_right(a_    ,	self.tokenizer.pad_token_id    )

         self.assertEqual(batch.input_ids.shape[1]    ,	3    )
         self.assertEqual(batch.decoder_input_ids.shape[1]    ,	10    )






   @require_torch
   def        _lowerCAmelCase    (		self   :		Any    ) ->						int:

         """simple docstring"""
         lowerCAmelCase__			: List[Any]  					=       self.tokenizer._build_translation_inputs(
             """A test"""    ,	return_tensors="""pt"""    ,	src_lang="""en_XX"""    ,	tgt_lang="""ar_AR"""    )

         self.assertEqual(
             nested_simplify(a_    )    ,	{
                 # A, test, EOS, en_XX
                 """input_ids""": [[62, 30_34, 2, 25_00_04]],
                 """attention_mask""": [[1, 1, 1, 1]],
                 # ar_AR
                 """forced_bos_token_id""": 25_00_01,
             }    ,	)









"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


SCREAMING_SNAKE_CASE :    Dict									=				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE :    Optional[int]									=				{
    """hustvl/yolos-small""": """https://huggingface.co/hustvl/yolos-small/resolve/main/config.json""",
    # See all YOLOS models at https://huggingface.co/models?filter=yolos
}



class 			_UpperCAmelCase (							__snake_case				):






		'''simple docstring'''

		lowerCamelCase__				='yolos'



		def __init__(self				,       a_=7_68				,       a_=12				,       a_=12				,       a_=30_72				,       a_="gelu"				,       a_=0.0				,       a_=0.0				,       a_=0.02				,       a_=1E-12				,       a_=[5_12, 8_64]				,       a_=16				,       a_=3				,       a_=True				,       a_=1_00				,       a_=True				,       a_=False				,       a_=1				,       a_=5				,       a_=2				,       a_=5				,       a_=2				,       a_=0.1				,       **a_				,       ):





			'''simple docstring'''

			super().__init__(**a_					)

			__snake_case	:						Union[str, Any]						=				hidden_size
			__snake_case	:						Union[str, Any]						=				num_hidden_layers
			__snake_case	:						Dict						=				num_attention_heads
			__snake_case	:						str						=				intermediate_size
			__snake_case	:						List[str]						=				hidden_act
			__snake_case	:						Tuple						=				hidden_dropout_prob
			__snake_case	:						Optional[int]						=				attention_probs_dropout_prob
			__snake_case	:						Union[str, Any]						=				initializer_range
			__snake_case	:						Tuple						=				layer_norm_eps
			__snake_case	:						List[Any]						=				image_size
			__snake_case	:						Tuple						=				patch_size
			__snake_case	:						str						=				num_channels
			__snake_case	:						Tuple						=				qkv_bias
			__snake_case	:						Union[str, Any]						=				num_detection_tokens
			__snake_case	:						List[str]						=				use_mid_position_embeddings
			__snake_case	:						Tuple						=				auxiliary_loss
			# Hungarian matcher
			__snake_case	:						List[str]						=				class_cost
			__snake_case	:						int						=				bbox_cost
			__snake_case	:						int						=				giou_cost
			# Loss coefficients
			__snake_case	:						Optional[int]						=				bbox_loss_coefficient
			__snake_case	:						List[str]						=				giou_loss_coefficient
			__snake_case	:						List[Any]						=				eos_coefficient



class 			_UpperCAmelCase (							__snake_case				):






		'''simple docstring'''

		lowerCamelCase__				=version.parse('1.11'				)



		@property
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			return OrderedDict(
			    [
			        ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
			    ]					)



		@property
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			return 1E-4



		@property
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			return 12





