


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE       :      Dict												= get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

SCREAMING_SNAKE_CASE       :      Union[str, Any]												= 256047
SCREAMING_SNAKE_CASE       :      str												= 256145






@require_sentencepiece
@require_tokenizers
class _lowerCamelCase(						_a, unittest.TestCase   ):
       lowercase_ :	Optional[Any]     =			NllbTokenizer
       lowercase_ :	str     =			NllbTokenizerFast
       lowercase_ :	Any     =			True
       lowercase_ :	Union[str, Any]     =			True
       lowercase_ :	Any     =			{}
       def 			UpperCamelCase     (					self)       ->   List[Any]:




           """simple docstring"""

           super().setUp()

           # We have a SentencePiece fixture for testing
           _lowercase       :							List[Any]      		=		NllbTokenizer(lowerCamelCase,      keep_accents=lowerCamelCase)
           tokenizer.save_pretrained(self.tmpdirname)
       def 			UpperCamelCase     (					self)       ->   Optional[Any]:




           """simple docstring"""

           _lowercase       :							Tuple      		=		NllbTokenizer(lowerCamelCase,      keep_accents=lowerCamelCase)

           _lowercase       :							Union[str, Any]      		=		tokenizer.tokenize('This is a test')
           self.assertListEqual(lowerCamelCase,      ['▁This', '▁is', '▁a', '▁t', 'est'])

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(lowerCamelCase),      [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]],      )

           _lowercase       :							List[str]      		=		tokenizer.tokenize('I was born in 92000, and this is falsé.')
           self.assertListEqual(
               lowerCamelCase,      [
                   SPIECE_UNDERLINE + 'I',
                   SPIECE_UNDERLINE + 'was',
                   SPIECE_UNDERLINE + 'b',
                   'or',
                   'n',
                   SPIECE_UNDERLINE + 'in',
                   SPIECE_UNDERLINE + '',
                   '9',
                   '2',
                   '0',
                   '0',
                   '0',
                   ',',
                   SPIECE_UNDERLINE + 'and',
                   SPIECE_UNDERLINE + 'this',
                   SPIECE_UNDERLINE + 'is',
                   SPIECE_UNDERLINE + 'f',
                   'al',
                   's',
                   'é',
                   '.',
               ],      )
           _lowercase       :							Optional[int]      		=		tokenizer.convert_tokens_to_ids(lowerCamelCase)
           self.assertListEqual(
               lowerCamelCase,      [
                   value + tokenizer.fairseq_offset
                   for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
               ],      )

           _lowercase       :							str      		=		tokenizer.convert_ids_to_tokens(lowerCamelCase)
           self.assertListEqual(
               lowerCamelCase,      [
                   SPIECE_UNDERLINE + 'I',
                   SPIECE_UNDERLINE + 'was',
                   SPIECE_UNDERLINE + 'b',
                   'or',
                   'n',
                   SPIECE_UNDERLINE + 'in',
                   SPIECE_UNDERLINE + '',
                   '<unk>',
                   '2',
                   '0',
                   '0',
                   '0',
                   ',',
                   SPIECE_UNDERLINE + 'and',
                   SPIECE_UNDERLINE + 'this',
                   SPIECE_UNDERLINE + 'is',
                   SPIECE_UNDERLINE + 'f',
                   'al',
                   's',
                   '<unk>',
                   '.',
               ],      )
       def 			UpperCamelCase     (					self)       ->   Tuple:




           """simple docstring"""

           _lowercase       :							Tuple      		=		(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
                   _lowercase       :							List[str]      		=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase,      **lowerCamelCase)
                   _lowercase       :							List[str]      		=		self.tokenizer_class.from_pretrained(lowerCamelCase,      **lowerCamelCase)

                   _lowercase       :							Union[str, Any]      		=		tempfile.mkdtemp()

                   _lowercase       :							Dict      		=		tokenizer_r.save_pretrained(lowerCamelCase)
                   _lowercase       :							Tuple      		=		tokenizer_p.save_pretrained(lowerCamelCase)

                   # Checks it save with the same files + the tokenizer.json file for the fast one
                   self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))
                   _lowercase       :							Tuple      		=		tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f)
                   self.assertSequenceEqual(lowerCamelCase,      lowerCamelCase)

                   # Checks everything loads correctly in the same way
                   _lowercase       :							int      		=		tokenizer_r.from_pretrained(lowerCamelCase)
                   _lowercase       :							Optional[Any]      		=		tokenizer_p.from_pretrained(lowerCamelCase)

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(lowerCamelCase,      lowerCamelCase))

                   shutil.rmtree(lowerCamelCase)

                   # Save tokenizer rust, legacy_format=True
                   _lowercase       :							List[str]      		=		tempfile.mkdtemp()

                   _lowercase       :							List[Any]      		=		tokenizer_r.save_pretrained(lowerCamelCase,      legacy_format=lowerCamelCase)
                   _lowercase       :							List[str]      		=		tokenizer_p.save_pretrained(lowerCamelCase)

                   # Checks it save with the same files
                   self.assertSequenceEqual(lowerCamelCase,      lowerCamelCase)

                   # Checks everything loads correctly in the same way
                   _lowercase       :							Optional[Any]      		=		tokenizer_r.from_pretrained(lowerCamelCase)
                   _lowercase       :							List[str]      		=		tokenizer_p.from_pretrained(lowerCamelCase)

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(lowerCamelCase,      lowerCamelCase))

                   shutil.rmtree(lowerCamelCase)

                   # Save tokenizer rust, legacy_format=False
                   _lowercase       :							Dict      		=		tempfile.mkdtemp()

                   _lowercase       :							Dict      		=		tokenizer_r.save_pretrained(lowerCamelCase,      legacy_format=lowerCamelCase)
                   _lowercase       :							Optional[int]      		=		tokenizer_p.save_pretrained(lowerCamelCase)

                   # Checks it saved the tokenizer.json file
                   self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))

                   # Checks everything loads correctly in the same way
                   _lowercase       :							str      		=		tokenizer_r.from_pretrained(lowerCamelCase)
                   _lowercase       :							Optional[int]      		=		tokenizer_p.from_pretrained(lowerCamelCase)

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(lowerCamelCase,      lowerCamelCase))

                   shutil.rmtree(lowerCamelCase)
       @require_torch
       def 			UpperCamelCase     (					self)       ->   Optional[Any]:




           """simple docstring"""

           if not self.test_seqaseq:
               return

           _lowercase       :							List[str]      		=		self.get_tokenizers()
           for tokenizer in tokenizers:
               with self.subTest(F'''{tokenizer.__class__.__name__}'''):
                   # Longer text that will definitely require truncation.
                   _lowercase       :							Optional[int]      		=		[
                       ' UN Chief Says There Is No Military Solution in Syria',
                       ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                       ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                       ' will only worsen the violence and misery for millions of people.',
                   ]
                   _lowercase       :							Optional[int]      		=		[
                       'Şeful ONU declară că nu există o soluţie militară în Siria',
                       'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                       ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                       ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                   ]
                   try:
                       _lowercase       :							str      		=		tokenizer.prepare_seqaseq_batch(
                           src_texts=lowerCamelCase,      tgt_texts=lowerCamelCase,      max_length=3,      max_target_length=10,      return_tensors='pt',      src_lang='eng_Latn',      tgt_lang='ron_Latn',      )
                   except NotImplementedError:
                       return
                   self.assertEqual(batch.input_ids.shape[1],      3)
                   self.assertEqual(batch.labels.shape[1],      10)
                   # max_target_length will default to max_length if not specified
                   _lowercase       :							Optional[Any]      		=		tokenizer.prepare_seqaseq_batch(
                       lowerCamelCase,      tgt_texts=lowerCamelCase,      max_length=3,      return_tensors='pt')
                   self.assertEqual(batch.input_ids.shape[1],      3)
                   self.assertEqual(batch.labels.shape[1],      3)

                   _lowercase       :							int      		=		tokenizer.prepare_seqaseq_batch(
                       src_texts=lowerCamelCase,      max_length=3,      max_target_length=10,      return_tensors='pt')
                   self.assertEqual(batch_encoder_only.input_ids.shape[1],      3)
                   self.assertEqual(batch_encoder_only.attention_mask.shape[1],      3)
                   self.assertNotIn('decoder_input_ids',      lowerCamelCase)
       @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.')
       def 			UpperCamelCase     (					self)       ->   Dict:




           """simple docstring"""

           pass




       def 			UpperCamelCase     (					self)       ->   Optional[int]:




           """simple docstring"""

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
                   _lowercase       :							int      		=		[AddedToken('<special>',      lstrip=lowerCamelCase)]

                   _lowercase       :							Dict      		=		self.rust_tokenizer_class.from_pretrained(
                       lowerCamelCase,      additional_special_tokens=lowerCamelCase,      **lowerCamelCase)
                   _lowercase       :							Optional[Any]      		=		tokenizer_r.encode('Hey this is a <special> token')

                   _lowercase       :							str      		=		tokenizer_r.encode('<special>',      add_special_tokens=lowerCamelCase)[0]

                   self.assertTrue(special_token_id in r_output)

                   if self.test_slow_tokenizer:
                       _lowercase       :							Tuple      		=		self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase,      additional_special_tokens=lowerCamelCase,      **lowerCamelCase,      )
                       _lowercase       :							str      		=		self.tokenizer_class.from_pretrained(
                           lowerCamelCase,      additional_special_tokens=lowerCamelCase,      **lowerCamelCase)

                       _lowercase       :							int      		=		tokenizer_p.encode('Hey this is a <special> token')

                       _lowercase       :							Optional[int]      		=		tokenizer_cr.encode('Hey this is a <special> token')

                       self.assertEqual(lowerCamelCase,      lowerCamelCase)
                       self.assertEqual(lowerCamelCase,      lowerCamelCase)
                       self.assertTrue(special_token_id in p_output)
                       self.assertTrue(special_token_id in cr_output)






@require_torch
@require_sentencepiece
@require_tokenizers
class _lowerCamelCase(						unittest.TestCase   ):
       lowercase_ :	Tuple     =			"""facebook/nllb-200-distilled-600M"""
       lowercase_ :	Any     =			[
           """ UN Chief Says There Is No Military Solution in Syria""",
           """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
       ]
       lowercase_ :	str     =			[
           """Şeful ONU declară că nu există o soluţie militară în Siria""",
           """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
           """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
           """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
       ]
       lowercase_ :	str     =			[
           25_60_47,
           1_62_97,
           13_44_08,
           81_65,
           24_80_66,
           1_47_34,
           9_50,
           11_35,
           10_57_21,
           35_73,
           83,
           2_73_52,
           1_08,
           4_94_86,
           2,
       ]
       @classmethod
       def 			UpperCamelCase     (					cls)       ->   Optional[int]:




           """simple docstring"""

           _lowercase       :							NllbTokenizer      		=		NllbTokenizer.from_pretrained(
               cls.checkpoint_name,      src_lang='eng_Latn',      tgt_lang='ron_Latn')
           _lowercase       :							str      		=		1
           return cls
       def 			UpperCamelCase     (					self)       ->   Union[str, Any]:




           """simple docstring"""

           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab'],      25_60_01)
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn'],      25_60_02)
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn'],      25_60_57)
       def 			UpperCamelCase     (					self)       ->   int:




           """simple docstring"""

           _lowercase       :							Dict      		=		self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
           self.assertListEqual(self.expected_src_tokens,      lowerCamelCase)
       def 			UpperCamelCase     (					self)       ->   Dict:




           """simple docstring"""

           self.assertIn(lowerCamelCase,      self.tokenizer.all_special_ids)
           # fmt: off
           _lowercase       :							int      		=		[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
           # fmt: on

           _lowercase       :							str      		=		self.tokenizer.decode(lowerCamelCase,      skip_special_tokens=lowerCamelCase)
           _lowercase       :							List[Any]      		=		self.tokenizer.decode(generated_ids[1:],      skip_special_tokens=lowerCamelCase)
           self.assertEqual(lowerCamelCase,      lowerCamelCase)
           self.assertNotIn(self.tokenizer.eos_token,      lowerCamelCase)
       def 			UpperCamelCase     (					self)       ->   Union[str, Any]:




           """simple docstring"""

           _lowercase       :							int      		=		['this is gunna be a long sentence ' * 20]
           assert isinstance(src_text[0],      lowerCamelCase)
           _lowercase       :							Any      		=		10
           _lowercase       :							int      		=		self.tokenizer(lowerCamelCase,      max_length=lowerCamelCase,      truncation=lowerCamelCase).input_ids[0]
           self.assertEqual(ids[-1],      2)
           self.assertEqual(ids[0],      lowerCamelCase)
           self.assertEqual(len(lowerCamelCase),      lowerCamelCase)
       def 			UpperCamelCase     (					self)       ->   Dict:




           """simple docstring"""

           self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']),      [25_62_03, 3])
       def 			UpperCamelCase     (					self)       ->   Optional[Any]:




           """simple docstring"""

           _lowercase       :							Any      		=		tempfile.mkdtemp()
           _lowercase       :							List[str]      		=		self.tokenizer.fairseq_tokens_to_ids
           self.tokenizer.save_pretrained(lowerCamelCase)
           _lowercase       :							Optional[int]      		=		NllbTokenizer.from_pretrained(lowerCamelCase)
           self.assertDictEqual(new_tok.fairseq_tokens_to_ids,      lowerCamelCase)
       @require_torch
       def 			UpperCamelCase     (					self)       ->   List[Any]:




           """simple docstring"""

           _lowercase       :							Union[str, Any]      		=		self.tokenizer(
               self.src_text,      text_target=self.tgt_text,      padding=lowerCamelCase,      truncation=lowerCamelCase,      max_length=len(self.expected_src_tokens),      return_tensors='pt',      )
           _lowercase       :							List[Any]      		=		shift_tokens_right(
               batch['labels'],      self.tokenizer.pad_token_id,      self.tokenizer.lang_code_to_id['ron_Latn'])

           self.assertIsInstance(lowerCamelCase,      lowerCamelCase)

           self.assertEqual((2, 15),      batch.input_ids.shape)
           self.assertEqual((2, 15),      batch.attention_mask.shape)
           _lowercase       :							Optional[Any]      		=		batch.input_ids.tolist()[0]
           self.assertListEqual(self.expected_src_tokens,      lowerCamelCase)
           self.assertEqual(lowerCamelCase,      batch.decoder_input_ids[0, 0])  # EOS
           # Test that special tokens are reset
           self.assertEqual(self.tokenizer.prefix_tokens,      [EN_CODE])
           self.assertEqual(self.tokenizer.suffix_tokens,      [self.tokenizer.eos_token_id])
       def 			UpperCamelCase     (					self)       ->   Tuple:




           """simple docstring"""

           _lowercase       :							Optional[int]      		=		self.tokenizer(self.src_text,      padding=lowerCamelCase,      truncation=lowerCamelCase,      max_length=3,      return_tensors='pt')
           _lowercase       :							Optional[Any]      		=		self.tokenizer(
               text_target=self.tgt_text,      padding=lowerCamelCase,      truncation=lowerCamelCase,      max_length=10,      return_tensors='pt')
           _lowercase       :							Tuple      		=		targets['input_ids']
           _lowercase       :							Any      		=		shift_tokens_right(
               lowerCamelCase,      self.tokenizer.pad_token_id,      decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang],      )

           self.assertEqual(batch.input_ids.shape[1],      3)
           self.assertEqual(batch.decoder_input_ids.shape[1],      10)
       @require_torch
       def 			UpperCamelCase     (					self)       ->   Optional[Any]:




           """simple docstring"""

           _lowercase       :							Union[str, Any]      		=		self.tokenizer._build_translation_inputs(
               'A test',      return_tensors='pt',      src_lang='eng_Latn',      tgt_lang='fra_Latn')

           self.assertEqual(
               nested_simplify(lowerCamelCase),      {
                   # A, test, EOS, en_XX
                   'input_ids': [[25_60_47, 70, 73_56, 2]],
                   'attention_mask': [[1, 1, 1, 1]],
                   # ar_AR
                   'forced_bos_token_id': 25_60_57,
               },      )




       @require_torch
       def 			UpperCamelCase     (					self)       ->   List[Any]:




           """simple docstring"""

           _lowercase       :							str      		=		True
           _lowercase       :							List[Any]      		=		self.tokenizer(
               'UN Chief says there is no military solution in Syria',      src_lang='eng_Latn',      tgt_lang='fra_Latn')
           self.assertEqual(
               inputs.input_ids,      [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47])

           _lowercase       :							Any      		=		False
           _lowercase       :							Dict      		=		self.tokenizer(
               'UN Chief says there is no military solution in Syria',      src_lang='eng_Latn',      tgt_lang='fra_Latn')
           self.assertEqual(
               inputs.input_ids,      [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2])





'''simple docstring'''






import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class 			lowercase_	(lowerCamelCase__      ):




  """simple docstring"""




  SCREAMING_SNAKE_CASE    :  Optional[Any]							=			['image_processor', 'tokenizer']
  SCREAMING_SNAKE_CASE    :  Union[str, Any]							=			'CLIPImageProcessor'
  SCREAMING_SNAKE_CASE    :  Union[str, Any]							=			('CLIPTokenizer', 'CLIPTokenizerFast')






  def __init__(      self   :     List[Any]					,lowercase__   :     Dict=None					,lowercase__   :     Union[str, Any]=None					,**lowercase__   :     Tuple			):
   __lowercase							  = None
   if "feature_extractor" in kwargs:
    warnings.warn(
        '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
        ''' instead.'''					,lowercase__					,)
    __lowercase							  = kwargs.pop('''feature_extractor'''			)

   __lowercase							  = image_processor if image_processor is not None else feature_extractor
   if image_processor is None:
    raise ValueError('''You need to specify an `image_processor`.'''			)
   if tokenizer is None:
    raise ValueError('''You need to specify a `tokenizer`.'''			)

   super().__init__(lowercase__					,lowercase__			)






  def __call__(      self   :     List[Any]					,lowercase__   :     str=None					,lowercase__   :     List[Any]=None					,lowercase__   :     Optional[Any]=None					,**lowercase__   :     int			):

   if text is None and images is None:
    raise ValueError('''You have to specify either text or images. Both cannot be none.'''			)

   if text is not None:
    __lowercase							  = self.tokenizer(lowercase__					,return_tensors=lowercase__					,**lowercase__			)

   if images is not None:
    __lowercase							  = self.image_processor(lowercase__					,return_tensors=lowercase__					,**lowercase__			)

   if text is not None and images is not None:
    __lowercase							  = image_features.pixel_values
    return encoding
   elif text is not None:
    return encoding
   else:
    return BatchEncoding(data=dict(**lowercase__			)					,tensor_type=lowercase__			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     Optional[int]					,*lowercase__   :     List[str]					,**lowercase__   :     int			):
   return self.tokenizer.batch_decode(*lowercase__					,**lowercase__			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     List[Any]					,*lowercase__   :     Optional[int]					,**lowercase__   :     Union[str, Any]			):
   return self.tokenizer.decode(*lowercase__					,**lowercase__			)






  @property
  def 					SCREAMING_SNAKE_CASE      (      self   :     Optional[Any]			):
   __lowercase							  = self.tokenizer.model_input_names
   __lowercase							  = self.image_processor.model_input_names
   return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names			)			)






  @property
  def 					SCREAMING_SNAKE_CASE      (      self   :     List[str]			):
   warnings.warn(
       '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''					,lowercase__					,)
   return self.image_processor_class






  @property
  def 					SCREAMING_SNAKE_CASE      (      self   :     Optional[int]			):
   warnings.warn(
       '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''					,lowercase__					,)
   return self.image_processor

