




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__  :  Dict					=       get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
      from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__magic_name__  :  Optional[int]					=       25_60_47
__magic_name__  :  Optional[int]					=       25_61_45





@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__		(_a    ,					unittest.TestCase			):
     lowercase_ :      Union[str, Any]				= NllbTokenizer
     lowercase_ :      Tuple				= NllbTokenizerFast
     lowercase_ :      Optional[int]				= True
     lowercase_ :      Optional[Any]				= True
     lowercase_ :      Any				= {}
     def        A__      (						self	:		str						):



       """simple docstring"""
       super().setUp()

       # We have a SentencePiece fixture for testing
       lowerCAmelCase__										=      NllbTokenizer(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)
       tokenizer.save_pretrained(self.tmpdirname						)
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      NllbTokenizer(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.tokenize('''This is a test'''						)
       self.assertListEqual(__lowerCamelCase       ,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']						)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(__lowerCamelCase						)       ,						[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]       ,						)

       lowerCAmelCase__										=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''						)
       self.assertListEqual(
           __lowerCamelCase       ,						[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ]       ,						)
       lowerCAmelCase__										=      tokenizer.convert_tokens_to_ids(__lowerCamelCase						)
       self.assertListEqual(
           __lowerCamelCase       ,						[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]       ,						)

       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(
           __lowerCamelCase       ,						[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''<unk>''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''<unk>''',
               '''.''',
           ]       ,						)
     def        A__      (						self	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
           lowerCAmelCase__										=      self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,						**__lowerCamelCase						)
           lowerCAmelCase__										=      self.tokenizer_class.from_pretrained(__lowerCamelCase       ,						**__lowerCamelCase						)

           lowerCAmelCase__										=      tempfile.mkdtemp()

           lowerCAmelCase__										=      tokenizer_r.save_pretrained(__lowerCamelCase						)
           lowerCAmelCase__										=      tokenizer_p.save_pretrained(__lowerCamelCase						)

           # Checks it save with the same files + the tokenizer.json file for the fast one
           self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files						)						)
           lowerCAmelCase__										=      tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f						)
           self.assertSequenceEqual(__lowerCamelCase       ,						__lowerCamelCase						)

           # Checks everything loads correctly in the same way
           lowerCAmelCase__										=      tokenizer_r.from_pretrained(__lowerCamelCase						)
           lowerCAmelCase__										=      tokenizer_p.from_pretrained(__lowerCamelCase						)

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__lowerCamelCase       ,						__lowerCamelCase						)						)

           shutil.rmtree(__lowerCamelCase						)

           # Save tokenizer rust, legacy_format=True
           lowerCAmelCase__										=      tempfile.mkdtemp()

           lowerCAmelCase__										=      tokenizer_r.save_pretrained(__lowerCamelCase       ,						legacy_format=__lowerCamelCase						)
           lowerCAmelCase__										=      tokenizer_p.save_pretrained(__lowerCamelCase						)

           # Checks it save with the same files
           self.assertSequenceEqual(__lowerCamelCase       ,						__lowerCamelCase						)

           # Checks everything loads correctly in the same way
           lowerCAmelCase__										=      tokenizer_r.from_pretrained(__lowerCamelCase						)
           lowerCAmelCase__										=      tokenizer_p.from_pretrained(__lowerCamelCase						)

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__lowerCamelCase       ,						__lowerCamelCase						)						)

           shutil.rmtree(__lowerCamelCase						)

           # Save tokenizer rust, legacy_format=False
           lowerCAmelCase__										=      tempfile.mkdtemp()

           lowerCAmelCase__										=      tokenizer_r.save_pretrained(__lowerCamelCase       ,						legacy_format=__lowerCamelCase						)
           lowerCAmelCase__										=      tokenizer_p.save_pretrained(__lowerCamelCase						)

           # Checks it saved the tokenizer.json file
           self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files						)						)

           # Checks everything loads correctly in the same way
           lowerCAmelCase__										=      tokenizer_r.from_pretrained(__lowerCamelCase						)
           lowerCAmelCase__										=      tokenizer_p.from_pretrained(__lowerCamelCase						)

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__lowerCamelCase       ,						__lowerCamelCase						)						)

           shutil.rmtree(__lowerCamelCase						)
     @require_torch
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       if not self.test_seqaseq:
         return

       lowerCAmelCase__										=      self.get_tokenizers()
       for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""						):
           # Longer text that will definitely require truncation.
           lowerCAmelCase__										=      [
               ''' UN Chief Says There Is No Military Solution in Syria''',
               ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
               ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
               ''' will only worsen the violence and misery for millions of people.''',
           ]
           lowerCAmelCase__										=      [
               '''Şeful ONU declară că nu există o soluţie militară în Siria''',
               '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
               ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
               ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
           ]
           try:
             lowerCAmelCase__										=      tokenizer.prepare_seqaseq_batch(
                 src_texts=__lowerCamelCase       ,						tgt_texts=__lowerCamelCase       ,						max_length=3       ,						max_target_length=10       ,						return_tensors='''pt'''       ,						src_lang='''eng_Latn'''       ,						tgt_lang='''ron_Latn'''       ,						)
           except NotImplementedError:
             return
           self.assertEqual(batch.input_ids.shape[1]       ,						3						)
           self.assertEqual(batch.labels.shape[1]       ,						10						)
           # max_target_length will default to max_length if not specified
           lowerCAmelCase__										=      tokenizer.prepare_seqaseq_batch(
               __lowerCamelCase       ,						tgt_texts=__lowerCamelCase       ,						max_length=3       ,						return_tensors='''pt'''						)
           self.assertEqual(batch.input_ids.shape[1]       ,						3						)
           self.assertEqual(batch.labels.shape[1]       ,						3						)

           lowerCAmelCase__										=      tokenizer.prepare_seqaseq_batch(
               src_texts=__lowerCamelCase       ,						max_length=3       ,						max_target_length=10       ,						return_tensors='''pt'''						)
           self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,						3						)
           self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,						3						)
           self.assertNotIn('''decoder_input_ids'''       ,						__lowerCamelCase						)
     @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       pass



     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
           lowerCAmelCase__										=      [AddedToken('''<special>'''       ,						lstrip=__lowerCamelCase						)]

           lowerCAmelCase__										=      self.rust_tokenizer_class.from_pretrained(
               __lowerCamelCase       ,						additional_special_tokens=__lowerCamelCase       ,						**__lowerCamelCase						)
           lowerCAmelCase__										=      tokenizer_r.encode('''Hey this is a <special> token'''						)

           lowerCAmelCase__										=      tokenizer_r.encode('''<special>'''       ,						add_special_tokens=__lowerCamelCase						)[0]

           self.assertTrue(special_token_id in r_output						)

           if self.test_slow_tokenizer:
             lowerCAmelCase__										=      self.rust_tokenizer_class.from_pretrained(
                 __lowerCamelCase       ,						additional_special_tokens=__lowerCamelCase       ,						**__lowerCamelCase       ,						)
             lowerCAmelCase__										=      self.tokenizer_class.from_pretrained(
                 __lowerCamelCase       ,						additional_special_tokens=__lowerCamelCase       ,						**__lowerCamelCase						)

             lowerCAmelCase__										=      tokenizer_p.encode('''Hey this is a <special> token'''						)

             lowerCAmelCase__										=      tokenizer_cr.encode('''Hey this is a <special> token'''						)

             self.assertEqual(__lowerCamelCase       ,						__lowerCamelCase						)
             self.assertEqual(__lowerCamelCase       ,						__lowerCamelCase						)
             self.assertTrue(special_token_id in p_output						)
             self.assertTrue(special_token_id in cr_output						)





@require_torch
@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__		(unittest.TestCase			):
     lowercase_ :      Tuple				= "facebook/nllb-200-distilled-600M"
     lowercase_ :      Tuple				= [
         " UN Chief Says There Is No Military Solution in Syria",
         " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
     ]
     lowercase_ :      Any				= [
         "Şeful ONU declară că nu există o soluţie militară în Siria",
         "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
         " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
         " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
     ]
     lowercase_ :      Optional[int]				= [
         256047,
         16297,
         134408,
         8165,
         248066,
         14734,
         950,
         1135,
         105721,
         3573,
         83,
         27352,
         108,
         49486,
         2,
     ]
     @classmethod
     def        A__      (						cls	:		Optional[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      NllbTokenizer.from_pretrained(
           cls.checkpoint_name       ,						src_lang='''eng_Latn'''       ,						tgt_lang='''ron_Latn'''						)
       lowerCAmelCase__										=      1
       return cls
     def        A__      (						self	:		int						):



       """simple docstring"""
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']       ,						25_60_01						)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']       ,						25_60_02						)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']       ,						25_60_57						)
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
       self.assertListEqual(self.expected_src_tokens       ,						__lowerCamelCase						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       self.assertIn(__lowerCamelCase       ,						self.tokenizer.all_special_ids						)
       # fmt: off
       lowerCAmelCase__										=      [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
       # fmt: on

       lowerCAmelCase__										=      self.tokenizer.decode(__lowerCamelCase       ,						skip_special_tokens=__lowerCamelCase						)
       lowerCAmelCase__										=      self.tokenizer.decode(generated_ids[1:]       ,						skip_special_tokens=__lowerCamelCase						)
       self.assertEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       self.assertNotIn(self.tokenizer.eos_token       ,						__lowerCamelCase						)
     def        A__      (						self	:		int						):



       """simple docstring"""
       lowerCAmelCase__										=      ['''this is gunna be a long sentence ''' * 20]
       assert isinstance(src_text[0]       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      10
       lowerCAmelCase__										=      self.tokenizer(__lowerCamelCase       ,						max_length=__lowerCamelCase       ,						truncation=__lowerCamelCase						).input_ids[0]
       self.assertEqual(ids[-1]       ,						2						)
       self.assertEqual(ids[0]       ,						__lowerCamelCase						)
       self.assertEqual(len(__lowerCamelCase						)       ,						__lowerCamelCase						)
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']						)       ,						[25_62_03, 3]						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       lowerCAmelCase__										=      tempfile.mkdtemp()
       lowerCAmelCase__										=      self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(__lowerCamelCase						)
       lowerCAmelCase__										=      NllbTokenizer.from_pretrained(__lowerCamelCase						)
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,						__lowerCamelCase						)
     @require_torch
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      self.tokenizer(
           self.src_text       ,						text_target=self.tgt_text       ,						padding=__lowerCamelCase       ,						truncation=__lowerCamelCase       ,						max_length=len(self.expected_src_tokens						)       ,						return_tensors='''pt'''       ,						)
       lowerCAmelCase__										=      shift_tokens_right(
           batch['''labels''']       ,						self.tokenizer.pad_token_id       ,						self.tokenizer.lang_code_to_id['''ron_Latn''']						)

       self.assertIsInstance(__lowerCamelCase       ,						__lowerCamelCase						)

       self.assertEqual((2, 15)       ,						batch.input_ids.shape						)
       self.assertEqual((2, 15)       ,						batch.attention_mask.shape						)
       lowerCAmelCase__										=      batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens       ,						__lowerCamelCase						)
       self.assertEqual(__lowerCamelCase       ,						batch.decoder_input_ids[0, 0]						)  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens       ,						[EN_CODE]						)
       self.assertEqual(self.tokenizer.suffix_tokens       ,						[self.tokenizer.eos_token_id]						)
     def        A__      (						self	:		Optional[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.tokenizer(self.src_text       ,						padding=__lowerCamelCase       ,						truncation=__lowerCamelCase       ,						max_length=3       ,						return_tensors='''pt'''						)
       lowerCAmelCase__										=      self.tokenizer(
           text_target=self.tgt_text       ,						padding=__lowerCamelCase       ,						truncation=__lowerCamelCase       ,						max_length=10       ,						return_tensors='''pt'''						)
       lowerCAmelCase__										=      targets['''input_ids''']
       lowerCAmelCase__										=      shift_tokens_right(
           __lowerCamelCase       ,						self.tokenizer.pad_token_id       ,						decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,						)

       self.assertEqual(batch.input_ids.shape[1]       ,						3						)
       self.assertEqual(batch.decoder_input_ids.shape[1]       ,						10						)
     @require_torch
     def        A__      (						self	:		int						):



       """simple docstring"""
       lowerCAmelCase__										=      self.tokenizer._build_translation_inputs(
           '''A test'''       ,						return_tensors='''pt'''       ,						src_lang='''eng_Latn'''       ,						tgt_lang='''fra_Latn'''						)

       self.assertEqual(
           nested_simplify(__lowerCamelCase						)       ,						{
               # A, test, EOS, en_XX
               '''input_ids''': [[25_60_47, 70, 73_56, 2]],
               '''attention_mask''': [[1, 1, 1, 1]],
               # ar_AR
               '''forced_bos_token_id''': 25_60_57,
           }       ,						)



     @require_torch
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      True
       lowerCAmelCase__										=      self.tokenizer(
           '''UN Chief says there is no military solution in Syria'''       ,						src_lang='''eng_Latn'''       ,						tgt_lang='''fra_Latn'''						)
       self.assertEqual(
           inputs.input_ids       ,						[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]						)

       lowerCAmelCase__										=      False
       lowerCAmelCase__										=      self.tokenizer(
           '''UN Chief says there is no military solution in Syria'''       ,						src_lang='''eng_Latn'''       ,						tgt_lang='''fra_Latn'''						)
       self.assertEqual(
           inputs.input_ids       ,						[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]						)







import torch
from torch import nn
from transformers import CLIPPreTrainedModel, CLIPVisionModel

from ...models.attention import BasicTransformerBlock
from ...utils import logging


__magic_name__  :  List[str]					=       logging.get_logger(__name__)  # pylint: disable=invalid-name





class     SCREAMING_SNAKE_CASE__		(_a			):
     def __init__(						self	:		List[Any]       ,						__lowerCamelCase	:		Tuple       ,						__lowerCamelCase	:		List[Any]=7_68						):



       """simple docstring"""
       super().__init__(__lowerCamelCase						)
       lowerCAmelCase__										=      proj_size

       lowerCAmelCase__										=      CLIPVisionModel(__lowerCamelCase						)
       lowerCAmelCase__										=      PaintByExampleMapper(__lowerCamelCase						)
       lowerCAmelCase__										=      nn.LayerNorm(config.hidden_size						)
       lowerCAmelCase__										=      nn.Linear(config.hidden_size       ,						self.proj_size						)

       # uncondition for scaling
       lowerCAmelCase__										=      nn.Parameter(torch.randn((1, 1, self.proj_size)						)						)



     def        A__      (						self	:		Tuple       ,						__lowerCamelCase	:		int       ,						__lowerCamelCase	:		str=False						):



       """simple docstring"""
       lowerCAmelCase__										=      self.model(pixel_values=__lowerCamelCase						)
       lowerCAmelCase__										=      clip_output.pooler_output
       lowerCAmelCase__										=      self.mapper(latent_states[:, None]						)
       lowerCAmelCase__										=      self.final_layer_norm(__lowerCamelCase						)
       lowerCAmelCase__										=      self.proj_out(__lowerCamelCase						)
       if return_uncond_vector:
         return latent_states, self.uncond_vector

       return latent_states





class     SCREAMING_SNAKE_CASE__		(nn.Module			):
     def __init__(						self	:		Optional[Any]       ,						__lowerCamelCase	:		Tuple						):



       """simple docstring"""
       super().__init__()
       lowerCAmelCase__										=      (config.num_hidden_layers + 1) // 5
       lowerCAmelCase__										=      config.hidden_size
       lowerCAmelCase__										=      1
       lowerCAmelCase__										=      nn.ModuleList(
           [
               BasicTransformerBlock(__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase       ,						activation_fn='''gelu'''       ,						attention_bias=__lowerCamelCase						)
               for _ in range(__lowerCamelCase						)
           ]						)



     def        A__      (						self	:		str       ,						__lowerCamelCase	:		Optional[Any]						):



       """simple docstring"""
       for block in self.blocks:
         lowerCAmelCase__										=      block(__lowerCamelCase						)

       return hidden_states

