

'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase :      Tuple					 =     logging.get_logger(__name__)

lowerCamelCase :      List[str]					 =     "▁"

lowerCamelCase :      Union[str, Any]					 =     {"vocab_file": "sentencepiece.bpe.model"}

lowerCamelCase :      Tuple					 =     {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

lowerCamelCase :      str					 =     {
    "facebook/mbart-large-50-one-to-many-mmt": 1_0_2_4,
}

# fmt: off
lowerCamelCase :      Dict					 =     ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]







class 							A__    (       A__  ):
  A__		 =				VOCAB_FILES_NAMES
  A__		 =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A__		 =				PRETRAINED_VOCAB_FILES_MAP
  A__		 =				['input_ids', 'attention_mask']

  A__		 =				[]
  A__		 =				[]





  def __init__(		self      :	int							,					_a      :	Tuple							,					_a      :	Optional[int]=None							,					_a      :	str=None							,					_a      :	Tuple="</s>"							,					_a      :	List[str]="</s>"							,					_a      :	Any="<s>"							,					_a      :	Dict="<unk>"							,					_a      :	Optional[Any]="<pad>"							,					_a      :	Optional[int]="<mask>"							,					_a      :	Optional[Dict[str, Any]] = None							,					**_a      :	List[Any]							,					)	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =AddedToken(_a							,					lstrip=_a							,					rstrip=_a ) if isinstance(_a							,					_a ) else mask_token

       _SCREAMING_SNAKE_CASE							    ={} if sp_model_kwargs is None else sp_model_kwargs

       _SCREAMING_SNAKE_CASE							    =kwargs.get('additional_special_tokens'							,					[] )
       kwargs["additional_special_tokens"] += [
           code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           src_lang=_a							,					tgt_lang=_a							,					eos_token=_a							,					unk_token=_a							,					sep_token=_a							,					cls_token=_a							,					pad_token=_a							,					mask_token=_a							,					sp_model_kwargs=self.sp_model_kwargs							,					**_a							,					)

       _SCREAMING_SNAKE_CASE							    =spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(str(_a ) )
       _SCREAMING_SNAKE_CASE							    =vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # Mimic fairseq token-to-id alignment for the first 4 token
       _SCREAMING_SNAKE_CASE							    ={'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       _SCREAMING_SNAKE_CASE							    =1

       _SCREAMING_SNAKE_CASE							    =len(self.sp_model )
       _SCREAMING_SNAKE_CASE							    ={
           code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_a )
       }
       _SCREAMING_SNAKE_CASE							    ={v: k for k, v in self.lang_code_to_id.items()}
       _SCREAMING_SNAKE_CASE							    =len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset

       self.fairseq_tokens_to_ids.update(self.lang_code_to_id )
       _SCREAMING_SNAKE_CASE							    ={v: k for k, v in self.fairseq_tokens_to_ids.items()}

       _SCREAMING_SNAKE_CASE							    =src_lang if src_lang is not None else 'en_XX'
       _SCREAMING_SNAKE_CASE							    =self.lang_code_to_id[self._src_lang]
       _SCREAMING_SNAKE_CASE							    =tgt_lang
       self.set_src_lang_special_tokens(self._src_lang )





  @property
  def 						A  (		self      :	Dict )	->    int:





       '''simple docstring'''






       return len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset + 1  # Plus 1 for the mask token





  @property
  def 						A  (		self      :	List[Any] )	->    str:





       '''simple docstring'''






       return self._src_lang





  @src_lang.setter
  def 						A  (		self      :	Optional[Any]							,					_a      :	str )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =new_src_lang
       self.set_src_lang_special_tokens(self._src_lang )





  def __getstate__(		self      :	List[Any] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.__dict__.copy()
       _SCREAMING_SNAKE_CASE							    =None
       return state





  def __setstate__(		self      :	Dict							,					_a      :	Dict )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =d

       # for backward compatibility
       if not hasattr(self							,					'sp_model_kwargs' ):
            _SCREAMING_SNAKE_CASE							    ={}

       _SCREAMING_SNAKE_CASE							    =spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(self.vocab_file )





  def 						A  (		self      :	List[Any] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ={self.convert_ids_to_tokens(_a ): i for i in range(self.vocab_size )}
       vocab.update(self.added_tokens_encoder )
       return vocab





  def 						A  (		self      :	Dict							,					_a      :	str )	->    List[str]:





       '''simple docstring'''






       return self.sp_model.encode(_a							,					out_type=_a )





  def 						A  (		self      :	str							,					_a      :	str )	->    int:





       '''simple docstring'''






       if token in self.fairseq_tokens_to_ids:
            return self.fairseq_tokens_to_ids[token]
       _SCREAMING_SNAKE_CASE							    =self.sp_model.PieceToId(_a )

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





  def 						A  (		self      :	Optional[int]							,					_a      :	int )	->    str:





       '''simple docstring'''






       if index in self.fairseq_ids_to_tokens:
            return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset )





  def 						A  (		self      :	List[Any]							,					_a      :	str )	->    int:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =[]
       _SCREAMING_SNAKE_CASE							    =''
       _SCREAMING_SNAKE_CASE							    =False
       for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                 if not prev_is_special:
                      out_string += " "
                 out_string += self.sp_model.decode(_a ) + token
                 _SCREAMING_SNAKE_CASE							    =True
                 _SCREAMING_SNAKE_CASE							    =[]
            else:
                 current_sub_tokens.append(_a )
                 _SCREAMING_SNAKE_CASE							    =False
       out_string += self.sp_model.decode(_a )
       return out_string.strip()





  def 						A  (		self      :	Optional[int]							,					_a      :	str							,					_a      :	Optional[str] = None )	->    Tuple[str]:





       '''simple docstring'''






       if not os.path.isdir(_a ):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory" )
            return
       _SCREAMING_SNAKE_CASE							    =os.path.join(
           _a							,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )

       if os.path.abspath(self.vocab_file ) != os.path.abspath(_a ) and os.path.isfile(self.vocab_file ):
            copyfile(self.vocab_file							,					_a )
       elif not os.path.isfile(self.vocab_file ):
            with open(_a							,					'wb' ) as fi:
                 _SCREAMING_SNAKE_CASE							    =self.sp_model.serialized_model_proto()
                 fi.write(_a )

       return (out_vocab_file,)





  def 						A  (		self      :	str							,					_a      :	List[int]							,					_a      :	Optional[List[int]] = None							,					_a      :	bool = False )	->    List[int]:





       '''simple docstring'''






       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=_a							,					token_ids_a=_a							,					already_has_special_tokens=_a )

       _SCREAMING_SNAKE_CASE							    =[1] * len(self.prefix_tokens )
       _SCREAMING_SNAKE_CASE							    =[1] * len(self.suffix_tokens )
       if token_ids_a is None:
            return prefix_ones + ([0] * len(_a )) + suffix_ones
       return prefix_ones + ([0] * len(_a )) + ([0] * len(_a )) + suffix_ones





  def 						A  (		self      :	List[str]							,					_a      :	List[int]							,					_a      :	Optional[List[int]] = None )	->    List[int]:





       '''simple docstring'''






       if token_ids_a is None:
            return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





  def 						A  (		self      :	Union[str, Any]							,					_a      :	Union[str, Any]							,					_a      :	str							,					_a      :	Optional[str]							,					_a      :	Optional[str]							,					**_a      :	Optional[Any] )	->    int:





       '''simple docstring'''






       if src_lang is None or tgt_lang is None:
            raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model' )
       _SCREAMING_SNAKE_CASE							    =src_lang
       _SCREAMING_SNAKE_CASE							    =self(_a							,					add_special_tokens=_a							,					return_tensors=_a							,					**_a )
       _SCREAMING_SNAKE_CASE							    =self.convert_tokens_to_ids(_a )
       _SCREAMING_SNAKE_CASE							    =tgt_lang_id
       return inputs





  def 						A  (		self      :	Optional[Any]							,					_a      :	List[str]							,					_a      :	str = "en_XX"							,					_a      :	Optional[List[str]] = None							,					_a      :	str = "ro_RO"							,					**_a      :	Optional[Any]							,					)	->    BatchEncoding:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =src_lang
       _SCREAMING_SNAKE_CASE							    =tgt_lang
       return super().prepare_seqaseq_batch(_a							,					_a							,					**_a )





  def 						A  (		self      :	int )	->    List[Any]:





       '''simple docstring'''






       return self.set_src_lang_special_tokens(self.src_lang )





  def 						A  (		self      :	Optional[Any] )	->    Optional[Any]:





       '''simple docstring'''






       return self.set_tgt_lang_special_tokens(self.tgt_lang )





  def 						A  (		self      :	Tuple							,					_a      :	str )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.lang_code_to_id[src_lang]
       _SCREAMING_SNAKE_CASE							    =[self.cur_lang_code_id]
       _SCREAMING_SNAKE_CASE							    =[self.eos_token_id]





  def 						A  (		self      :	Optional[int]							,					_a      :	str )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.lang_code_to_id[tgt_lang]
       _SCREAMING_SNAKE_CASE							    =[self.cur_lang_code_id]
       _SCREAMING_SNAKE_CASE							    =[self.eos_token_id]








"""simple docstring"""






import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test




class 							lowerCAmelCase__				(			unittest.TestCase ):






      '''simple docstring'''

      @parameterized.expand([(None,), ('''foo.json''',)])
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		GenerationConfig(
              do_sample=lowercase_						,	temperature=0.7						,	length_penalty=1.0						,	bad_words_ids=[[1, 2, 3], [4, 5]]						,	)
          with tempfile.TemporaryDirectory() as tmp_dir:
              config.save_pretrained(lowercase_						,	config_name=lowercase_)
              SCREAMING_SNAKE_CASE_      :  Tuple    						=		GenerationConfig.from_pretrained(lowercase_						,	config_name=lowercase_)

          # Checks parameters that were specified
          self.assertEqual(loaded_config.do_sample						,	lowercase_)
          self.assertEqual(loaded_config.temperature						,	0.7)
          self.assertEqual(loaded_config.length_penalty						,	1.0)
          self.assertEqual(loaded_config.bad_words_ids						,	[[1, 2, 3], [4, 5]])

          # Checks parameters that were not specified (defaults)
          self.assertEqual(loaded_config.top_k						,	50)
          self.assertEqual(loaded_config.max_length						,	20)
          self.assertEqual(loaded_config.max_time						,	lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		AutoConfig.from_pretrained('''gpt2''')
          SCREAMING_SNAKE_CASE_      :  int    						=		GenerationConfig.from_model_config(lowercase_)
          SCREAMING_SNAKE_CASE_      :  int    						=		GenerationConfig()

          # The generation config has loaded a few non-default parameters from the model config
          self.assertNotEqual(lowercase_						,	lowercase_)

          # One of those parameters is eos_token_id -- check if it matches
          self.assertNotEqual(generation_config_from_model.eos_token_id						,	default_generation_config.eos_token_id)
          self.assertEqual(generation_config_from_model.eos_token_id						,	model_config.eos_token_id)
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		GenerationConfig()
          SCREAMING_SNAKE_CASE_      :  Any    						=		{
              '''max_new_tokens''': 1024,
              '''foo''': '''bar''',
          }
          SCREAMING_SNAKE_CASE_      :  str    						=		copy.deepcopy(lowercase_)
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		generation_config.update(**lowercase_)

          # update_kwargs was not modified (no side effects)
          self.assertEqual(lowercase_						,	lowercase_)

          # update_kwargs was used to update the config on valid attributes
          self.assertEqual(generation_config.max_new_tokens						,	1024)

          # `.update()` returns a dictionary of unused kwargs
          self.assertEqual(lowercase_						,	{'''foo''': '''bar'''})
      def _SCREAMING_SNAKE_CASE (						self     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		GenerationConfig()
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		'''bar'''

          with tempfile.TemporaryDirectory('''test-generation-config''') as tmp_dir:
              generation_config.save_pretrained(lowercase_)

              SCREAMING_SNAKE_CASE_      :  List[Any]    						=		GenerationConfig.from_pretrained(lowercase_)
          # update_kwargs was used to update the config on valid attributes
          self.assertEqual(new_config.foo						,	'''bar''')

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		GenerationConfig.from_model_config(lowercase_)
          assert not hasattr(lowercase_						,	'''foo''')  # no new kwargs should be initialized if from config



      def _SCREAMING_SNAKE_CASE (						self     :  Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		GenerationConfig()
          self.assertEqual(default_config.temperature						,	1.0)
          self.assertEqual(default_config.do_sample						,	lowercase_)
          self.assertEqual(default_config.num_beams						,	1)

          SCREAMING_SNAKE_CASE_      :  Dict    						=		GenerationConfig(
              do_sample=lowercase_						,	temperature=0.7						,	length_penalty=1.0						,	bad_words_ids=[[1, 2, 3], [4, 5]]						,	)
          self.assertEqual(config.temperature						,	0.7)
          self.assertEqual(config.do_sample						,	lowercase_)
          self.assertEqual(config.num_beams						,	1)

          with tempfile.TemporaryDirectory() as tmp_dir:
              config.save_pretrained(lowercase_)
              SCREAMING_SNAKE_CASE_      :  Tuple    						=		GenerationConfig.from_pretrained(lowercase_						,	temperature=1.0)

          self.assertEqual(loaded_config.temperature						,	1.0)
          self.assertEqual(loaded_config.do_sample						,	lowercase_)
          self.assertEqual(loaded_config.num_beams						,	1)  # default value




@is_staging_test
class 							lowerCAmelCase__				(			unittest.TestCase ):






      '''simple docstring'''

      @classmethod
      def _SCREAMING_SNAKE_CASE (						cls     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		TOKEN
          HfFolder.save_token(lowercase_)
      @classmethod
      def _SCREAMING_SNAKE_CASE (						cls     :  List[str]):




          '''simple docstring'''


          try:
              delete_repo(token=cls._token						,	repo_id='''test-generation-config''')
          except HTTPError:
              pass

          try:
              delete_repo(token=cls._token						,	repo_id='''valid_org/test-generation-config-org''')
          except HTTPError:
              pass
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		GenerationConfig(
              do_sample=lowercase_						,	temperature=0.7						,	length_penalty=1.0						,	)
          config.push_to_hub('''test-generation-config'''						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  int    						=		GenerationConfig.from_pretrained(F'{USER}/test-generation-config')
          for k, v in config.to_dict().items():
              if k != "transformers_version":
                  self.assertEqual(lowercase_						,	getattr(lowercase_						,	lowercase_))

        # Reset repo
          delete_repo(token=self._token						,	repo_id='''test-generation-config''')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
              config.save_pretrained(
                  lowercase_						,	repo_id='''test-generation-config'''						,	push_to_hub=lowercase_						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		GenerationConfig.from_pretrained(F'{USER}/test-generation-config')
          for k, v in config.to_dict().items():
              if k != "transformers_version":
                  self.assertEqual(lowercase_						,	getattr(lowercase_						,	lowercase_))



      def _SCREAMING_SNAKE_CASE (						self     :  Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		GenerationConfig(
              do_sample=lowercase_						,	temperature=0.7						,	length_penalty=1.0						,	)
          config.push_to_hub('''valid_org/test-generation-config-org'''						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  Any    						=		GenerationConfig.from_pretrained('''valid_org/test-generation-config-org''')
          for k, v in config.to_dict().items():
              if k != "transformers_version":
                  self.assertEqual(lowercase_						,	getattr(lowercase_						,	lowercase_))

        # Reset repo
          delete_repo(token=self._token						,	repo_id='''valid_org/test-generation-config-org''')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
              config.save_pretrained(
                  lowercase_						,	repo_id='''valid_org/test-generation-config-org'''						,	push_to_hub=lowercase_						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  Any    						=		GenerationConfig.from_pretrained('''valid_org/test-generation-config-org''')
          for k, v in config.to_dict().items():
              if k != "transformers_version":
                  self.assertEqual(lowercase_						,	getattr(lowercase_						,	lowercase_))


