"""simple docstring"""







import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE    	=get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
       from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__SCREAMING_SNAKE_CASE    	=25_6047
__SCREAMING_SNAKE_CASE    	=25_6145







@require_sentencepiece
@require_tokenizers
class        UpperCamelCase		(			_snake_case						,      unittest.TestCase  ):
   lowercase							=  NllbTokenizer
   lowercase							=  NllbTokenizerFast
   lowercase							=  True
   lowercase							=  True
   lowercase							=  {}
   def 			_UpperCAmelCase   (			self			)	->      List[str]:


        '''simple docstring'''


        super().setUp()

        # We have a SentencePiece fixture for testing
        lowercase_	:				List[Any]													=	NllbTokenizer(_lowerCamelCase     ,keep_accents=_lowerCamelCase			)
        tokenizer.save_pretrained(self.tmpdirname			)
   def 			_UpperCAmelCase   (			self			)	->      Optional[int]:


        '''simple docstring'''


        lowercase_	:				Any													=	NllbTokenizer(_lowerCamelCase     ,keep_accents=_lowerCamelCase			)

        lowercase_	:				List[Any]													=	tokenizer.tokenize('This is a test'			)
        self.assertListEqual(_lowerCamelCase     ,['▁This', '▁is', '▁a', '▁t', 'est']			)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(_lowerCamelCase			)     ,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,)

        lowercase_	:				Any													=	tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
        self.assertListEqual(
            _lowerCamelCase     ,[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ]     ,)
        lowercase_	:				Any													=	tokenizer.convert_tokens_to_ids(_lowerCamelCase			)
        self.assertListEqual(
            _lowerCamelCase     ,[
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ]     ,)

        lowercase_	:				Any													=	tokenizer.convert_ids_to_tokens(_lowerCamelCase			)
        self.assertListEqual(
            _lowerCamelCase     ,[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ]     ,)
   def 			_UpperCAmelCase   (			self			)	->      Any:


        '''simple docstring'''


        lowercase_	:				Any													=	(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
                  lowercase_	:				Any													=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase			)
                  lowercase_	:				Any													=	self.tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase			)

                  lowercase_	:				List[Any]													=	tempfile.mkdtemp()

                  lowercase_	:				List[Any]													=	tokenizer_r.save_pretrained(_lowerCamelCase			)
                  lowercase_	:				Optional[int]													=	tokenizer_p.save_pretrained(_lowerCamelCase			)

                  # Checks it save with the same files + the tokenizer.json file for the fast one
                  self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files			)			)
                  lowercase_	:				List[str]													=	tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f			)
                  self.assertSequenceEqual(_lowerCamelCase     ,_lowerCamelCase			)

                  # Checks everything loads correctly in the same way
                  lowercase_	:				str													=	tokenizer_r.from_pretrained(_lowerCamelCase			)
                  lowercase_	:				str													=	tokenizer_p.from_pretrained(_lowerCamelCase			)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(_lowerCamelCase     ,_lowerCamelCase			)			)

                  shutil.rmtree(_lowerCamelCase			)

                  # Save tokenizer rust, legacy_format=True
                  lowercase_	:				int													=	tempfile.mkdtemp()

                  lowercase_	:				str													=	tokenizer_r.save_pretrained(_lowerCamelCase     ,legacy_format=_lowerCamelCase			)
                  lowercase_	:				List[Any]													=	tokenizer_p.save_pretrained(_lowerCamelCase			)

                  # Checks it save with the same files
                  self.assertSequenceEqual(_lowerCamelCase     ,_lowerCamelCase			)

                  # Checks everything loads correctly in the same way
                  lowercase_	:				List[Any]													=	tokenizer_r.from_pretrained(_lowerCamelCase			)
                  lowercase_	:				Optional[int]													=	tokenizer_p.from_pretrained(_lowerCamelCase			)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(_lowerCamelCase     ,_lowerCamelCase			)			)

                  shutil.rmtree(_lowerCamelCase			)

                  # Save tokenizer rust, legacy_format=False
                  lowercase_	:				Optional[int]													=	tempfile.mkdtemp()

                  lowercase_	:				str													=	tokenizer_r.save_pretrained(_lowerCamelCase     ,legacy_format=_lowerCamelCase			)
                  lowercase_	:				List[str]													=	tokenizer_p.save_pretrained(_lowerCamelCase			)

                  # Checks it saved the tokenizer.json file
                  self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files			)			)

                  # Checks everything loads correctly in the same way
                  lowercase_	:				Tuple													=	tokenizer_r.from_pretrained(_lowerCamelCase			)
                  lowercase_	:				Optional[Any]													=	tokenizer_p.from_pretrained(_lowerCamelCase			)

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(_lowerCamelCase     ,_lowerCamelCase			)			)

                  shutil.rmtree(_lowerCamelCase			)
   @require_torch
   def 			_UpperCAmelCase   (			self			)	->      List[Any]:


        '''simple docstring'''


        if not self.test_seqaseq:
             return

        lowercase_	:				Union[str, Any]													=	self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(f'''{tokenizer.__class__.__name__}'''			):
                  # Longer text that will definitely require truncation.
                  lowercase_	:				int													=	[
                      ''' UN Chief Says There Is No Military Solution in Syria''',
                      ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                      ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                      ''' will only worsen the violence and misery for millions of people.''',
                  ]
                  lowercase_	:				Tuple													=	[
                      '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                      '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                      ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                      ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                  ]
                  try:
                       lowercase_	:				Optional[Any]													=	tokenizer.prepare_seqaseq_batch(
                           src_texts=_lowerCamelCase     ,tgt_texts=_lowerCamelCase     ,max_length=3     ,max_target_length=10     ,return_tensors='pt'     ,src_lang='eng_Latn'     ,tgt_lang='ron_Latn'     ,)
                  except NotImplementedError:
                       return
                  self.assertEqual(batch.input_ids.shape[1]     ,3			)
                  self.assertEqual(batch.labels.shape[1]     ,10			)
                  # max_target_length will default to max_length if not specified
                  lowercase_	:				List[Any]													=	tokenizer.prepare_seqaseq_batch(
                      _lowerCamelCase     ,tgt_texts=_lowerCamelCase     ,max_length=3     ,return_tensors='pt'			)
                  self.assertEqual(batch.input_ids.shape[1]     ,3			)
                  self.assertEqual(batch.labels.shape[1]     ,3			)

                  lowercase_	:				int													=	tokenizer.prepare_seqaseq_batch(
                      src_texts=_lowerCamelCase     ,max_length=3     ,max_target_length=10     ,return_tensors='pt'			)
                  self.assertEqual(batch_encoder_only.input_ids.shape[1]     ,3			)
                  self.assertEqual(batch_encoder_only.attention_mask.shape[1]     ,3			)
                  self.assertNotIn('decoder_input_ids'     ,_lowerCamelCase			)
   @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'			)
   def 			_UpperCAmelCase   (			self			)	->      Tuple:


        '''simple docstring'''


        pass







   def 			_UpperCAmelCase   (			self			)	->      Optional[Any]:


        '''simple docstring'''


        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
                  lowercase_	:				Optional[Any]													=	[AddedToken('<special>'     ,lstrip=_lowerCamelCase			)]

                  lowercase_	:				List[Any]													=	self.rust_tokenizer_class.from_pretrained(
                      _lowerCamelCase     ,additional_special_tokens=_lowerCamelCase     ,**_lowerCamelCase			)
                  lowercase_	:				List[str]													=	tokenizer_r.encode('Hey this is a <special> token'			)

                  lowercase_	:				Optional[Any]													=	tokenizer_r.encode('<special>'     ,add_special_tokens=_lowerCamelCase			)[0]

                  self.assertTrue(special_token_id in r_output			)

                  if self.test_slow_tokenizer:
                       lowercase_	:				int													=	self.rust_tokenizer_class.from_pretrained(
                           _lowerCamelCase     ,additional_special_tokens=_lowerCamelCase     ,**_lowerCamelCase     ,)
                       lowercase_	:				str													=	self.tokenizer_class.from_pretrained(
                           _lowerCamelCase     ,additional_special_tokens=_lowerCamelCase     ,**_lowerCamelCase			)

                       lowercase_	:				Any													=	tokenizer_p.encode('Hey this is a <special> token'			)

                       lowercase_	:				Tuple													=	tokenizer_cr.encode('Hey this is a <special> token'			)

                       self.assertEqual(_lowerCamelCase     ,_lowerCamelCase			)
                       self.assertEqual(_lowerCamelCase     ,_lowerCamelCase			)
                       self.assertTrue(special_token_id in p_output			)
                       self.assertTrue(special_token_id in cr_output			)







@require_torch
@require_sentencepiece
@require_tokenizers
class        UpperCamelCase		(			unittest.TestCase  ):
   lowercase							=  'facebook/nllb-200-distilled-600M'
   lowercase							=  [
       ' UN Chief Says There Is No Military Solution in Syria',
       ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
   ]
   lowercase							=  [
       'Şeful ONU declară că nu există o soluţie militară în Siria',
       'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
       ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
       ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
   ]
   lowercase							=  [
       2_5_6_0_4_7,
       1_6_2_9_7,
       1_3_4_4_0_8,
       8_1_6_5,
       2_4_8_0_6_6,
       1_4_7_3_4,
       9_5_0,
       1_1_3_5,
       1_0_5_7_2_1,
       3_5_7_3,
       8_3,
       2_7_3_5_2,
       1_0_8,
       4_9_4_8_6,
       2,
   ]
   @classmethod
   def 			_UpperCAmelCase   (			cls			)	->      List[Any]:


        '''simple docstring'''


        lowercase_	:				NllbTokenizer													=	NllbTokenizer.from_pretrained(
            cls.checkpoint_name     ,src_lang='eng_Latn'     ,tgt_lang='ron_Latn'			)
        lowercase_	:				Union[str, Any]													=	1
        return cls
   def 			_UpperCAmelCase   (			self			)	->      Tuple:


        '''simple docstring'''


        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']     ,25_6001			)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']     ,25_6002			)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']     ,25_6057			)
   def 			_UpperCAmelCase   (			self			)	->      Dict:


        '''simple docstring'''


        lowercase_	:				Any													=	self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
        self.assertListEqual(self.expected_src_tokens     ,_lowerCamelCase			)
   def 			_UpperCAmelCase   (			self			)	->      int:


        '''simple docstring'''


        self.assertIn(_lowerCamelCase     ,self.tokenizer.all_special_ids			)
        # fmt: off
        lowercase_	:				Union[str, Any]													=	[RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
        # fmt: on

        lowercase_	:				List[Any]													=	self.tokenizer.decode(_lowerCamelCase     ,skip_special_tokens=_lowerCamelCase			)
        lowercase_	:				Any													=	self.tokenizer.decode(generated_ids[1:]     ,skip_special_tokens=_lowerCamelCase			)
        self.assertEqual(_lowerCamelCase     ,_lowerCamelCase			)
        self.assertNotIn(self.tokenizer.eos_token     ,_lowerCamelCase			)
   def 			_UpperCAmelCase   (			self			)	->      Optional[int]:


        '''simple docstring'''


        lowercase_	:				Optional[int]													=	['''this is gunna be a long sentence ''' * 20]
        assert isinstance(src_text[0]     ,_lowerCamelCase			)
        lowercase_	:				Dict													=	10
        lowercase_	:				int													=	self.tokenizer(_lowerCamelCase     ,max_length=_lowerCamelCase     ,truncation=_lowerCamelCase			).input_ids[0]
        self.assertEqual(ids[-1]     ,2			)
        self.assertEqual(ids[0]     ,_lowerCamelCase			)
        self.assertEqual(len(_lowerCamelCase			)     ,_lowerCamelCase			)
   def 			_UpperCAmelCase   (			self			)	->      Any:


        '''simple docstring'''


        self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']			)     ,[25_6203, 3]			)
   def 			_UpperCAmelCase   (			self			)	->      str:


        '''simple docstring'''


        lowercase_	:				str													=	tempfile.mkdtemp()
        lowercase_	:				Dict													=	self.tokenizer.fairseq_tokens_to_ids
        self.tokenizer.save_pretrained(_lowerCamelCase			)
        lowercase_	:				int													=	NllbTokenizer.from_pretrained(_lowerCamelCase			)
        self.assertDictEqual(new_tok.fairseq_tokens_to_ids     ,_lowerCamelCase			)
   @require_torch
   def 			_UpperCAmelCase   (			self			)	->      List[Any]:


        '''simple docstring'''


        lowercase_	:				List[str]													=	self.tokenizer(
            self.src_text     ,text_target=self.tgt_text     ,padding=_lowerCamelCase     ,truncation=_lowerCamelCase     ,max_length=len(self.expected_src_tokens			)     ,return_tensors='pt'     ,)
        lowercase_	:				str													=	shift_tokens_right(
            batch['labels']     ,self.tokenizer.pad_token_id     ,self.tokenizer.lang_code_to_id['ron_Latn']			)

        self.assertIsInstance(_lowerCamelCase     ,_lowerCamelCase			)

        self.assertEqual((2, 15)     ,batch.input_ids.shape			)
        self.assertEqual((2, 15)     ,batch.attention_mask.shape			)
        lowercase_	:				str													=	batch.input_ids.tolist()[0]
        self.assertListEqual(self.expected_src_tokens     ,_lowerCamelCase			)
        self.assertEqual(_lowerCamelCase     ,batch.decoder_input_ids[0, 0]			)  # EOS
        # Test that special tokens are reset
        self.assertEqual(self.tokenizer.prefix_tokens     ,[EN_CODE]			)
        self.assertEqual(self.tokenizer.suffix_tokens     ,[self.tokenizer.eos_token_id]			)
   def 			_UpperCAmelCase   (			self			)	->      Union[str, Any]:


        '''simple docstring'''


        lowercase_	:				Tuple													=	self.tokenizer(self.src_text     ,padding=_lowerCamelCase     ,truncation=_lowerCamelCase     ,max_length=3     ,return_tensors='pt'			)
        lowercase_	:				Any													=	self.tokenizer(
            text_target=self.tgt_text     ,padding=_lowerCamelCase     ,truncation=_lowerCamelCase     ,max_length=10     ,return_tensors='pt'			)
        lowercase_	:				List[str]													=	targets['''input_ids''']
        lowercase_	:				Union[str, Any]													=	shift_tokens_right(
            _lowerCamelCase     ,self.tokenizer.pad_token_id     ,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]     ,)

        self.assertEqual(batch.input_ids.shape[1]     ,3			)
        self.assertEqual(batch.decoder_input_ids.shape[1]     ,10			)
   @require_torch
   def 			_UpperCAmelCase   (			self			)	->      List[str]:


        '''simple docstring'''


        lowercase_	:				List[str]													=	self.tokenizer._build_translation_inputs(
            'A test'     ,return_tensors='pt'     ,src_lang='eng_Latn'     ,tgt_lang='fra_Latn'			)

        self.assertEqual(
            nested_simplify(_lowerCamelCase			)     ,{
                # A, test, EOS, en_XX
                'input_ids': [[25_6047, 70, 7356, 2]],
                'attention_mask': [[1, 1, 1, 1]],
                # ar_AR
                'forced_bos_token_id': 25_6057,
            }     ,)







   @require_torch
   def 			_UpperCAmelCase   (			self			)	->      List[Any]:


        '''simple docstring'''


        lowercase_	:				Optional[Any]													=	True
        lowercase_	:				int													=	self.tokenizer(
            'UN Chief says there is no military solution in Syria'     ,src_lang='eng_Latn'     ,tgt_lang='fra_Latn'			)
        self.assertEqual(
            inputs.input_ids     ,[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]			)

        lowercase_	:				Optional[int]													=	False
        lowercase_	:				Optional[Any]													=	self.tokenizer(
            'UN Chief says there is no military solution in Syria'     ,src_lang='eng_Latn'     ,tgt_lang='fra_Latn'			)
        self.assertEqual(
            inputs.input_ids     ,[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]			)





import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

snake_case				:    Union[str, Any]       				=     '''python tqdm regex requests packaging filelock numpy tokenizers'''.split()
if sys.version_info < (3, 7):
     pkgs_to_check_at_runtime.append('''dataclasses''')
if sys.version_info < (3, 8):
     pkgs_to_check_at_runtime.append('''importlib_metadata''')

for pkg in pkgs_to_check_at_runtime:
     if pkg in deps:
          if pkg == "tokenizers":
               # must be loaded here, or else tqdm check may fail
               from .utils import is_tokenizers_available

               if not is_tokenizers_available():
                    continue  # not required, check version only if installed

          require_version_core(deps[pkg])
     else:
          raise ValueError(F"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		Optional[int]=None						):




      """simple docstring"""





      require_version(deps[pkg] ,  UpperCAmelCase_						)





