
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__		:     str     			=      get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
   from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase__		:     Any     			=      256047
UpperCAmelCase__		:     Optional[Any]     			=      256145






@require_sentencepiece
@require_tokenizers
class UpperCAmelCase   (						SCREAMING_SNAKE_CASE__							,							unittest.TestCase   ):





   '''simple docstring'''
   __UpperCamelCase		:    str 				=    NllbTokenizer
   __UpperCamelCase		:    Dict 				=    NllbTokenizerFast
   __UpperCamelCase		:    List[str] 				=    True
   __UpperCamelCase		:    Dict 				=    True
   __UpperCamelCase		:    List[str] 				=    {}


   def  __magic_name__				(					self     :							str     ):

      """simple docstring"""
      super().setUp()

      # We have a SentencePiece fixture for testing
      _A:  str					=       NllbTokenizer(lowerCAmelCase_			,			keep_accents=lowerCAmelCase_     )
      tokenizer.save_pretrained(self.tmpdirname     )


   def  __magic_name__				(					self     :							int     ):

      """simple docstring"""
      _A:  Dict					=       NllbTokenizer(lowerCAmelCase_			,			keep_accents=lowerCAmelCase_     )

      _A:  str					=       tokenizer.tokenize('''This is a test'''     )
      self.assertListEqual(lowerCAmelCase_			,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']     )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )			,			[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]			,			)

      _A:  Tuple					=       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''     )
      self.assertListEqual(
          lowerCAmelCase_			,			[
              SPIECE_UNDERLINE + '''I''',
              SPIECE_UNDERLINE + '''was''',
              SPIECE_UNDERLINE + '''b''',
              '''or''',
              '''n''',
              SPIECE_UNDERLINE + '''in''',
              SPIECE_UNDERLINE + '''''',
              '''9''',
              '''2''',
              '''0''',
              '''0''',
              '''0''',
              ''',''',
              SPIECE_UNDERLINE + '''and''',
              SPIECE_UNDERLINE + '''this''',
              SPIECE_UNDERLINE + '''is''',
              SPIECE_UNDERLINE + '''f''',
              '''al''',
              '''s''',
              '''é''',
              '''.''',
          ]			,			)
      _A:  Dict					=       tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )
      self.assertListEqual(
          lowerCAmelCase_			,			[
              value + tokenizer.fairseq_offset
              for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
          ]			,			)

      _A:  Optional[int]					=       tokenizer.convert_ids_to_tokens(lowerCAmelCase_     )
      self.assertListEqual(
          lowerCAmelCase_			,			[
              SPIECE_UNDERLINE + '''I''',
              SPIECE_UNDERLINE + '''was''',
              SPIECE_UNDERLINE + '''b''',
              '''or''',
              '''n''',
              SPIECE_UNDERLINE + '''in''',
              SPIECE_UNDERLINE + '''''',
              '''<unk>''',
              '''2''',
              '''0''',
              '''0''',
              '''0''',
              ''',''',
              SPIECE_UNDERLINE + '''and''',
              SPIECE_UNDERLINE + '''this''',
              SPIECE_UNDERLINE + '''is''',
              SPIECE_UNDERLINE + '''f''',
              '''al''',
              '''s''',
              '''<unk>''',
              '''.''',
          ]			,			)


   def  __magic_name__				(					self     :							int     ):

      """simple docstring"""
      _A:  List[Any]					=       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
            _A:  Optional[Any]					=       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,			**lowerCAmelCase_     )
            _A:  Any					=       self.tokenizer_class.from_pretrained(lowerCAmelCase_			,			**lowerCAmelCase_     )

            _A:  Dict					=       tempfile.mkdtemp()

            _A:  Optional[int]					=       tokenizer_r.save_pretrained(lowerCAmelCase_     )
            _A:  Dict					=       tokenizer_p.save_pretrained(lowerCAmelCase_     )

            # Checks it save with the same files + the tokenizer.json file for the fast one
            self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files     )     )
            _A:  Optional[Any]					=       tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f     )
            self.assertSequenceEqual(lowerCAmelCase_			,			lowerCAmelCase_     )

            # Checks everything loads correctly in the same way
            _A:  Union[str, Any]					=       tokenizer_r.from_pretrained(lowerCAmelCase_     )
            _A:  str					=       tokenizer_p.from_pretrained(lowerCAmelCase_     )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(lowerCAmelCase_			,			lowerCAmelCase_     )     )

            shutil.rmtree(lowerCAmelCase_     )

            # Save tokenizer rust, legacy_format=True
            _A:  Union[str, Any]					=       tempfile.mkdtemp()

            _A:  Dict					=       tokenizer_r.save_pretrained(lowerCAmelCase_			,			legacy_format=lowerCAmelCase_     )
            _A:  int					=       tokenizer_p.save_pretrained(lowerCAmelCase_     )

            # Checks it save with the same files
            self.assertSequenceEqual(lowerCAmelCase_			,			lowerCAmelCase_     )

            # Checks everything loads correctly in the same way
            _A:  Optional[int]					=       tokenizer_r.from_pretrained(lowerCAmelCase_     )
            _A:  Dict					=       tokenizer_p.from_pretrained(lowerCAmelCase_     )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(lowerCAmelCase_			,			lowerCAmelCase_     )     )

            shutil.rmtree(lowerCAmelCase_     )

            # Save tokenizer rust, legacy_format=False
            _A:  Tuple					=       tempfile.mkdtemp()

            _A:  int					=       tokenizer_r.save_pretrained(lowerCAmelCase_			,			legacy_format=lowerCAmelCase_     )
            _A:  Dict					=       tokenizer_p.save_pretrained(lowerCAmelCase_     )

            # Checks it saved the tokenizer.json file
            self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files     )     )

            # Checks everything loads correctly in the same way
            _A:  Optional[Any]					=       tokenizer_r.from_pretrained(lowerCAmelCase_     )
            _A:  Union[str, Any]					=       tokenizer_p.from_pretrained(lowerCAmelCase_     )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(lowerCAmelCase_			,			lowerCAmelCase_     )     )

            shutil.rmtree(lowerCAmelCase_     )


   @require_torch
   def  __magic_name__				(					self     :							Dict     ):

      """simple docstring"""
      if not self.test_seqaseq:
         return

      _A:  Any					=       self.get_tokenizers()
      for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""     ):
            # Longer text that will definitely require truncation.
            _A:  List[Any]					=       [
                ''' UN Chief Says There Is No Military Solution in Syria''',
                ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                ''' will only worsen the violence and misery for millions of people.''',
            ]
            _A:  int					=       [
                '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
            ]
            try:
               _A:  List[Any]					=       tokenizer.prepare_seqaseq_batch(
                   src_texts=lowerCAmelCase_			,			tgt_texts=lowerCAmelCase_			,			max_length=3			,			max_target_length=1_0			,			return_tensors='''pt'''			,			src_lang='''eng_Latn'''			,			tgt_lang='''ron_Latn'''			,			)
            except NotImplementedError:
               return
            self.assertEqual(batch.input_ids.shape[1]			,			3     )
            self.assertEqual(batch.labels.shape[1]			,			1_0     )
            # max_target_length will default to max_length if not specified
            _A:  Optional[int]					=       tokenizer.prepare_seqaseq_batch(
                lowerCAmelCase_			,			tgt_texts=lowerCAmelCase_			,			max_length=3			,			return_tensors='''pt'''     )
            self.assertEqual(batch.input_ids.shape[1]			,			3     )
            self.assertEqual(batch.labels.shape[1]			,			3     )

            _A:  str					=       tokenizer.prepare_seqaseq_batch(
                src_texts=lowerCAmelCase_			,			max_length=3			,			max_target_length=1_0			,			return_tensors='''pt'''     )
            self.assertEqual(batch_encoder_only.input_ids.shape[1]			,			3     )
            self.assertEqual(batch_encoder_only.attention_mask.shape[1]			,			3     )
            self.assertNotIn('''decoder_input_ids'''			,			lowerCAmelCase_     )


   @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''     )
   def  __magic_name__				(					self     :							int     ):

      """simple docstring"""
      pass


   def  __magic_name__				(					self     :							List[str]     ):

      """simple docstring"""
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
            _A:  int					=       [AddedToken('''<special>'''			,			lstrip=lowerCAmelCase_     )]

            _A:  Optional[Any]					=       self.rust_tokenizer_class.from_pretrained(
                lowerCAmelCase_			,			additional_special_tokens=lowerCAmelCase_			,			**lowerCAmelCase_     )
            _A:  Optional[int]					=       tokenizer_r.encode('''Hey this is a <special> token'''     )

            _A:  List[Any]					=       tokenizer_r.encode('''<special>'''			,			add_special_tokens=lowerCAmelCase_     )[0]

            self.assertTrue(special_token_id in r_output     )

            if self.test_slow_tokenizer:
               _A:  Optional[int]					=       self.rust_tokenizer_class.from_pretrained(
                   lowerCAmelCase_			,			additional_special_tokens=lowerCAmelCase_			,			**lowerCAmelCase_			,			)
               _A:  Any					=       self.tokenizer_class.from_pretrained(
                   lowerCAmelCase_			,			additional_special_tokens=lowerCAmelCase_			,			**lowerCAmelCase_     )

               _A:  Dict					=       tokenizer_p.encode('''Hey this is a <special> token'''     )

               _A:  Optional[int]					=       tokenizer_cr.encode('''Hey this is a <special> token'''     )

               self.assertEqual(lowerCAmelCase_			,			lowerCAmelCase_     )
               self.assertEqual(lowerCAmelCase_			,			lowerCAmelCase_     )
               self.assertTrue(special_token_id in p_output     )
               self.assertTrue(special_token_id in cr_output     )






@require_torch
@require_sentencepiece
@require_tokenizers
class UpperCAmelCase   (						unittest.TestCase   ):





   '''simple docstring'''
   __UpperCamelCase		:    Dict 				=    '''facebook/nllb-200-distilled-600M'''
   __UpperCamelCase		:    int 				=    [
       ''' UN Chief Says There Is No Military Solution in Syria''',
       ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
   ]
   __UpperCamelCase		:    Any 				=    [
       '''Şeful ONU declară că nu există o soluţie militară în Siria''',
       '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
       ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
       ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
   ]
   __UpperCamelCase		:    Any 				=    [
       25_6047,
       1_6297,
       13_4408,
       8165,
       24_8066,
       1_4734,
       950,
       1135,
       10_5721,
       3573,
       83,
       2_7352,
       108,
       4_9486,
       2,
   ]


   @classmethod
   def  __magic_name__				(					cls     :							List[str]     ):

      """simple docstring"""
      _A:  NllbTokenizer					=       NllbTokenizer.from_pretrained(
          cls.checkpoint_name			,			src_lang='''eng_Latn'''			,			tgt_lang='''ron_Latn'''     )
      _A:  List[str]					=       1
      return cls


   def  __magic_name__				(					self     :							List[str]     ):

      """simple docstring"""
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']			,			2_5_6_0_0_1     )
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']			,			2_5_6_0_0_2     )
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']			,			2_5_6_0_5_7     )


   def  __magic_name__				(					self     :							Tuple     ):

      """simple docstring"""
      _A:  Tuple					=       self.tokenizer.batch_encode_plus(self.src_text     ).input_ids[0]
      self.assertListEqual(self.expected_src_tokens			,			lowerCAmelCase_     )


   def  __magic_name__				(					self     :							Dict     ):

      """simple docstring"""
      self.assertIn(lowerCAmelCase_			,			self.tokenizer.all_special_ids     )
      # fmt: off
      _A:  int					=       [RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
      # fmt: on

      _A:  str					=       self.tokenizer.decode(lowerCAmelCase_			,			skip_special_tokens=lowerCAmelCase_     )
      _A:  Tuple					=       self.tokenizer.decode(generated_ids[1:]			,			skip_special_tokens=lowerCAmelCase_     )
      self.assertEqual(lowerCAmelCase_			,			lowerCAmelCase_     )
      self.assertNotIn(self.tokenizer.eos_token			,			lowerCAmelCase_     )


   def  __magic_name__				(					self     :							List[str]     ):

      """simple docstring"""
      _A:  Optional[Any]					=       ['''this is gunna be a long sentence ''' * 2_0]
      assert isinstance(src_text[0]			,			lowerCAmelCase_     )
      _A:  Dict					=       1_0
      _A:  List[Any]					=       self.tokenizer(lowerCAmelCase_			,			max_length=lowerCAmelCase_			,			truncation=lowerCAmelCase_     ).input_ids[0]
      self.assertEqual(ids[-1]			,			2     )
      self.assertEqual(ids[0]			,			lowerCAmelCase_     )
      self.assertEqual(len(lowerCAmelCase_     )			,			lowerCAmelCase_     )


   def  __magic_name__				(					self     :							int     ):

      """simple docstring"""
      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']     )			,			[2_5_6_2_0_3, 3]     )


   def  __magic_name__				(					self     :							Any     ):

      """simple docstring"""
      _A:  Any					=       tempfile.mkdtemp()
      _A:  Union[str, Any]					=       self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(lowerCAmelCase_     )
      _A:  List[str]					=       NllbTokenizer.from_pretrained(lowerCAmelCase_     )
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,			lowerCAmelCase_     )


   @require_torch
   def  __magic_name__				(					self     :							Dict     ):

      """simple docstring"""
      _A:  int					=       self.tokenizer(
          self.src_text			,			text_target=self.tgt_text			,			padding=lowerCAmelCase_			,			truncation=lowerCAmelCase_			,			max_length=len(self.expected_src_tokens     )			,			return_tensors='''pt'''			,			)
      _A:  List[Any]					=       shift_tokens_right(
          batch['''labels''']			,			self.tokenizer.pad_token_id			,			self.tokenizer.lang_code_to_id['''ron_Latn''']     )

      self.assertIsInstance(lowerCAmelCase_			,			lowerCAmelCase_     )

      self.assertEqual((2, 1_5)			,			batch.input_ids.shape     )
      self.assertEqual((2, 1_5)			,			batch.attention_mask.shape     )
      _A:  List[Any]					=       batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens			,			lowerCAmelCase_     )
      self.assertEqual(lowerCAmelCase_			,			batch.decoder_input_ids[0, 0]     )  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens			,			[EN_CODE]     )
      self.assertEqual(self.tokenizer.suffix_tokens			,			[self.tokenizer.eos_token_id]     )


   def  __magic_name__				(					self     :							str     ):

      """simple docstring"""
      _A:  int					=       self.tokenizer(self.src_text			,			padding=lowerCAmelCase_			,			truncation=lowerCAmelCase_			,			max_length=3			,			return_tensors='''pt'''     )
      _A:  Any					=       self.tokenizer(
          text_target=self.tgt_text			,			padding=lowerCAmelCase_			,			truncation=lowerCAmelCase_			,			max_length=1_0			,			return_tensors='''pt'''     )
      _A:  Optional[Any]					=       targets['''input_ids''']
      _A:  Union[str, Any]					=       shift_tokens_right(
          lowerCAmelCase_			,			self.tokenizer.pad_token_id			,			decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]			,			)

      self.assertEqual(batch.input_ids.shape[1]			,			3     )
      self.assertEqual(batch.decoder_input_ids.shape[1]			,			1_0     )


   @require_torch
   def  __magic_name__				(					self     :							Tuple     ):

      """simple docstring"""
      _A:  Tuple					=       self.tokenizer._build_translation_inputs(
          '''A test'''			,			return_tensors='''pt'''			,			src_lang='''eng_Latn'''			,			tgt_lang='''fra_Latn'''     )

      self.assertEqual(
          nested_simplify(lowerCAmelCase_     )			,			{
              # A, test, EOS, en_XX
              '''input_ids''': [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
              '''attention_mask''': [[1, 1, 1, 1]],
              # ar_AR
              '''forced_bos_token_id''': 2_5_6_0_5_7,
          }			,			)


   @require_torch
   def  __magic_name__				(					self     :							str     ):

      """simple docstring"""
      _A:  List[Any]					=       True
      _A:  List[Any]					=       self.tokenizer(
          '''UN Chief says there is no military solution in Syria'''			,			src_lang='''eng_Latn'''			,			tgt_lang='''fra_Latn'''     )
      self.assertEqual(
          inputs.input_ids			,			[1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7]     )

      _A:  Dict					=       False
      _A:  Tuple					=       self.tokenizer(
          '''UN Chief says there is no military solution in Syria'''			,			src_lang='''eng_Latn'''			,			tgt_lang='''fra_Latn'''     )
      self.assertEqual(
          inputs.input_ids			,			[2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2]     )





def   lowerCamelCase__							(					a		,		a      )      ->   str:
   if a < 0 or b < 0:
      raise ValueError('''the value of both inputs must be positive'''      )

   _A:  Union[str, Any]					=       str(bin(a      )      )[2:]  # remove the leading "0b"
   _A:  Union[str, Any]					=       str(bin(a      )      )[2:]  # remove the leading "0b"

   _A:  Optional[int]					=       max(len(a      )		,		len(a      )      )

   return "0b" + "".join(
       str(int(char_a != char_b      )      )
       for char_a, char_b in zip(a_binary.zfill(a      )		,		b_binary.zfill(a      )      )      )


if __name__ == "__main__":
   import doctest

   doctest.testmod()



