



'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_nllb import NllbTokenizer
else:
     lowerCAmelCase:					str						       =						None


lowerCAmelCase:					Optional[Any]						       =						logging.get_logger(__name__)


lowerCAmelCase:					List[Any]						       =						{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase:					List[Any]						       =						{
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase:					List[Any]						       =						{
    """facebook/nllb-large-en-ro""": 1_0_2_4,
    """facebook/nllb-200-distilled-600M""": 1_0_2_4,
}

# fmt: off
lowerCAmelCase:					Tuple						       =						["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]

class 							a__(					__UpperCAmelCase	):
    lowercase__ =    VOCAB_FILES_NAMES
    lowercase__ =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__ =    PRETRAINED_VOCAB_FILES_MAP
    lowercase__ =    ['''input_ids''', '''attention_mask''']
    lowercase__ =    NllbTokenizer

    lowercase__ =    []
    lowercase__ =    []







    def __init__(      self :  int						,			__snake_case :  str=None						,			__snake_case :  Dict=None						,			__snake_case :  Dict="<s>"						,			__snake_case :  int="</s>"						,			__snake_case :  Any="</s>"						,			__snake_case :  int="<s>"						,			__snake_case :  Dict="<unk>"						,			__snake_case :  Tuple="<pad>"						,			__snake_case :  Optional[Any]="<mask>"						,			__snake_case :  Optional[Any]=None						,			__snake_case :  List[str]=None						,			__snake_case :  Union[str, Any]=None						,			__snake_case :  Dict=False						,			**__snake_case :  Any						,			):
      a						:					Optional[Any]        =     AddedToken(__snake_case						,			lstrip=__snake_case						,			rstrip=__snake_case	) if isinstance(__snake_case						,			__snake_case	) else mask_token
      a						:					str        =     legacy_behaviour
      super().__init__(
          vocab_file=__snake_case						,			tokenizer_file=__snake_case						,			bos_token=__snake_case						,			eos_token=__snake_case						,			sep_token=__snake_case						,			cls_token=__snake_case						,			unk_token=__snake_case						,			pad_token=__snake_case						,			mask_token=__snake_case						,			src_lang=__snake_case						,			tgt_lang=__snake_case						,			additional_special_tokens=__snake_case						,			legacy_behaviour=__snake_case						,			**__snake_case						,			)

      a						:					Optional[int]        =     vocab_file
      a						:					str        =     False if not self.vocab_file else True

      a						:					List[Any]        =     FAIRSEQ_LANGUAGE_CODES.copy()

      if additional_special_tokens is not None:
        # Only add those special tokens if they are not already there.
        _additional_special_tokens.extend(
            [t for t in additional_special_tokens if t not in _additional_special_tokens]	)

      self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}	)
      a						:					Any        =     {
          lang_code: self.convert_tokens_to_ids(__snake_case	) for lang_code in FAIRSEQ_LANGUAGE_CODES
      }

      a						:					str        =     src_lang if src_lang is not None else 'eng_Latn'
      a						:					int        =     self.convert_tokens_to_ids(self._src_lang	)
      a						:					Any        =     tgt_lang
      self.set_src_lang_special_tokens(self._src_lang	)







    @property
    def 			lowercase_       (      self :  List[str]	):
      return self._src_lang







    @src_lang.setter
    def 			lowercase_       (      self :  Tuple						,			__snake_case :  Optional[Any]	):
      a						:					Tuple        =     new_src_lang
      self.set_src_lang_special_tokens(self._src_lang	)







    def 			lowercase_       (      self :  Union[str, Any]						,			__snake_case :  Dict						,			__snake_case :  Any = None	):

      if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







    def 			lowercase_       (      self :  str						,			__snake_case :  Dict						,			__snake_case :  str = None	):
      a						:					Optional[int]        =     [self.sep_token_id]
      a						:					Optional[Any]        =     [self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep	) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]







    def 			lowercase_       (      self :  List[str]						,			__snake_case :  List[Any]						,			__snake_case :  int						,			__snake_case :  int						,			__snake_case :  Union[str, Any]						,			**__snake_case :  List[Any]	):

      if src_lang is None or tgt_lang is None:
        raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
      a						:					List[str]        =     src_lang
      a						:					Any        =     self(__snake_case						,			add_special_tokens=__snake_case						,			return_tensors=__snake_case						,			**__snake_case	)
      a						:					Optional[int]        =     self.convert_tokens_to_ids(__snake_case	)
      a						:					List[str]        =     tgt_lang_id
      return inputs







    def 			lowercase_       (      self :  Tuple						,			__snake_case :  Tuple						,			__snake_case :  List[str] = "eng_Latn"						,			__snake_case :  Optional[int] = None						,			__snake_case :  List[Any] = "fra_Latn"						,			**__snake_case :  List[Any]						,			):
      a						:					Any        =     src_lang
      a						:					str        =     tgt_lang
      return super().prepare_seqaseq_batch(__snake_case						,			__snake_case						,			**__snake_case	)







    def 			lowercase_       (      self :  Union[str, Any]	):
      return self.set_src_lang_special_tokens(self.src_lang	)







    def 			lowercase_       (      self :  Dict	):
      return self.set_tgt_lang_special_tokens(self.tgt_lang	)







    def 			lowercase_       (      self :  Optional[int]						,			__snake_case :  Optional[int]	):
      a						:					List[str]        =     self.convert_tokens_to_ids(__snake_case	)

      if self.legacy_behaviour:
        a						:					Optional[Any]        =     []
        a						:					Optional[Any]        =     [self.eos_token_id, self.cur_lang_code]
      else:
        a						:					int        =     [self.cur_lang_code]
        a						:					Union[str, Any]        =     [self.eos_token_id]

      a						:					List[Any]        =     self.convert_ids_to_tokens(self.prefix_tokens	)
      a						:					Optional[Any]        =     self.convert_ids_to_tokens(self.suffix_tokens	)

      a						:					Optional[int]        =     processors.TemplateProcessing(
          single=prefix_tokens_str + ['$A'] + suffix_tokens_str						,			pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str						,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,			self.prefix_tokens + self.suffix_tokens	)	)						,			)







    def 			lowercase_       (      self :  List[str]						,			__snake_case :  List[Any]	):
      a						:					Optional[Any]        =     self.convert_tokens_to_ids(__snake_case	)
      if self.legacy_behaviour:
        a						:					int        =     []
        a						:					Union[str, Any]        =     [self.eos_token_id, self.cur_lang_code]
      else:
        a						:					Dict        =     [self.cur_lang_code]
        a						:					List[Any]        =     [self.eos_token_id]

      a						:					Optional[int]        =     self.convert_ids_to_tokens(self.prefix_tokens	)
      a						:					int        =     self.convert_ids_to_tokens(self.suffix_tokens	)

      a						:					int        =     processors.TemplateProcessing(
          single=prefix_tokens_str + ['$A'] + suffix_tokens_str						,			pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str						,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,			self.prefix_tokens + self.suffix_tokens	)	)						,			)







    def 			lowercase_       (      self :  Optional[int]						,			__snake_case :  List[Any]						,			__snake_case :  List[str] = None	):

      if not self.can_save_slow_tokenizer:
        raise ValueError(
            'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
            'tokenizer.'	)

      if not os.path.isdir(__snake_case	):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""	)
        return
      a						:					Any        =     os.path.join(
          __snake_case						,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

      if os.path.abspath(self.vocab_file	) != os.path.abspath(__snake_case	):
        copyfile(self.vocab_file						,			__snake_case	)

      return (out_vocab_file,)






import inspect
import unittest

from transformers import ViTMSNConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import ViTMSNForImageClassification, ViTMSNModel
    from transformers.models.vit_msn.modeling_vit_msn import VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    from PIL import Image

    from transformers import ViTImageProcessor





class  A__    :




   """simple docstring"""



   def __init__(					self					,  lowercase					,  lowercase=13					,  lowercase=30					,  lowercase=2					,  lowercase=3					,  lowercase=True					,  lowercase=True					,  lowercase=32					,  lowercase=5					,  lowercase=4					,  lowercase=37					,  lowercase="gelu"					,  lowercase=0.1					,  lowercase=0.1					,  lowercase=10					,  lowercase=0.02					,  lowercase=None					,  )			->						Union[str, Any]:







          '''simple docstring'''

          a__						:	Optional[int] =				parent
          a__						:	List[str] =				batch_size
          a__						:	List[str] =				image_size
          a__						:	Dict =				patch_size
          a__						:	Optional[Any] =				num_channels
          a__						:	List[Any] =				is_training
          a__						:	str =				use_labels
          a__						:	Dict =				hidden_size
          a__						:	Tuple =				num_hidden_layers
          a__						:	Tuple =				num_attention_heads
          a__						:	Union[str, Any] =				intermediate_size
          a__						:	List[str] =				hidden_act
          a__						:	List[str] =				hidden_dropout_prob
          a__						:	Any =				attention_probs_dropout_prob
          a__						:	Dict =				type_sequence_label_size
          a__						:	Tuple =				initializer_range
          a__						:	Optional[int] =				scope

          # in ViT MSN, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
          a__						:	List[str] =				(image_size // patch_size) ** 2
          a__						:	Any =				num_patches + 1

   def 							__lowercase		(					self)			->						int:







          '''simple docstring'''

          a__						:	Tuple =				floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

          a__						:	Tuple =				None
          if self.use_labels:
                 a__						:	List[Any] =				ids_tensor([self.batch_size]					,  self.type_sequence_label_size)

          a__						:	List[str] =				self.get_config()

          return config, pixel_values, labels

   def 							__lowercase		(					self)			->						Optional[int]:







          '''simple docstring'''

          return ViTMSNConfig(
              image_size=self.image_size					,  patch_size=self.patch_size					,  num_channels=self.num_channels					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  initializer_range=self.initializer_range					,  )

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase)			->						List[str]:







          '''simple docstring'''

          a__						:	int =				ViTMSNModel(config=lowercase)
          model.to(lowercase)
          model.eval()
          a__						:	int =				model(lowercase)
          self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size))

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase)			->						Tuple:







          '''simple docstring'''

          a__						:	Optional[Any] =				self.type_sequence_label_size
          a__						:	List[str] =				ViTMSNForImageClassification(lowercase)
          model.to(lowercase)
          model.eval()
          a__						:	int =				model(lowercase					,  labels=lowercase)
          print('Pixel and labels shape: {pixel_values.shape}, {labels.shape}')
          print('Labels: {labels}')
          self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.type_sequence_label_size))

          # test greyscale images
          a__						:	List[str] =				1
          a__						:	Optional[int] =				ViTMSNForImageClassification(lowercase)
          model.to(lowercase)
          model.eval()

          a__						:	Dict =				floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
          a__						:	Dict =				model(lowercase)
          self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.type_sequence_label_size))

   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          a__						:	List[Any] =				self.prepare_config_and_inputs()
          a__ ,							a__ ,							a__						:	Optional[Any] =				config_and_inputs
          a__						:	Dict =				{'pixel_values': pixel_values}
          return config, inputs_dict





@require_torch
class  A__    (							__UpperCAmelCase ,  __UpperCAmelCase ,  unittest.TestCase	):




   """simple docstring"""



   __A   :				Any         =  (ViTMSNModel, ViTMSNForImageClassification) if is_torch_available() else ()
   __A   :				Tuple         =  (
       {'''feature-extraction''': ViTMSNModel, '''image-classification''': ViTMSNForImageClassification}
       if is_torch_available()
       else {}
   )

   __A   :				List[str]         =  False
   __A   :				Optional[Any]         =  False
   __A   :				Union[str, Any]         =  False
   __A   :				Any         =  False

   def 							__lowercase		(					self)			->						List[str]:







          '''simple docstring'''

          a__						:	Optional[int] =				ViTMSNModelTester(self)
          a__						:	Union[str, Any] =				ConfigTester(self					,  config_class=lowercase					,  has_text_modality=lowercase					,  hidden_size=37)

   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          self.config_tester.run_common_tests()

   @unittest.skip(reason='ViTMSN does not use inputs_embeds')
   def 							__lowercase		(					self)			->						Any:







          '''simple docstring'''

          pass

   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          a__ ,							a__						:	Tuple =				self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
                 a__						:	Union[str, Any] =				model_class(lowercase)
                 self.assertIsInstance(model.get_input_embeddings()					,  (nn.Module))
                 a__						:	str =				model.get_output_embeddings()
                 self.assertTrue(x is None or isinstance(lowercase					,  nn.Linear))

   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          a__ ,							a__						:	Optional[int] =				self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
                 a__						:	Dict =				model_class(lowercase)
                 a__						:	List[Any] =				inspect.signature(model.forward)
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 a__						:	str =				[*signature.parameters.keys()]

                 a__						:	Optional[Any] =				['pixel_values']
                 self.assertListEqual(arg_names[:1]					,  lowercase)

   def 							__lowercase		(					self)			->						str:







          '''simple docstring'''

          a__						:	List[str] =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*lowercase)

   def 							__lowercase		(					self)			->						str:







          '''simple docstring'''

          a__						:	Any =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_image_classification(*lowercase)

   @slow
   def 							__lowercase		(					self)			->						Union[str, Any]:







          '''simple docstring'''

          for model_name in VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 a__						:	Tuple =				ViTMSNModel.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)



def A_	(		)  ->		Dict:
       a__						:	List[str] =				Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'				)
       return image





@require_torch
@require_vision
class  A__    (							unittest.TestCase	):




   """simple docstring"""



   @cached_property
   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          return ViTImageProcessor.from_pretrained('facebook/vit-msn-small') if is_vision_available() else None

   @slow
   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          torch.manual_seed(2)
          a__						:	List[str] =				ViTMSNForImageClassification.from_pretrained('facebook/vit-msn-small').to(lowercase)

          a__						:	Any =				self.default_image_processor
          a__						:	List[Any] =				prepare_img()
          a__						:	Optional[Any] =				image_processor(images=lowercase					,  return_tensors='pt').to(lowercase)

          # forward pass
          with torch.no_grad():
                 a__						:	Tuple =				model(**lowercase)

          # verify the logits
          a__						:	Union[str, Any] =				torch.Size((1, 1000))
          self.assertEqual(outputs.logits.shape					,  lowercase)

          a__						:	Any =				torch.tensor([-0.08_03, -0.44_54, -0.23_75]).to(lowercase)

          self.assertTrue(torch.allclose(outputs.logits[0, :3]					,  lowercase					,  atol=1e-4))


