

'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase :      Tuple					 =     logging.get_logger(__name__)

lowerCamelCase :      List[str]					 =     "▁"

lowerCamelCase :      Union[str, Any]					 =     {"vocab_file": "sentencepiece.bpe.model"}

lowerCamelCase :      Tuple					 =     {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

lowerCamelCase :      str					 =     {
    "facebook/mbart-large-50-one-to-many-mmt": 1_0_2_4,
}

# fmt: off
lowerCamelCase :      Dict					 =     ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]







class 							A__    (       A__  ):
  A__		 =				VOCAB_FILES_NAMES
  A__		 =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A__		 =				PRETRAINED_VOCAB_FILES_MAP
  A__		 =				['input_ids', 'attention_mask']

  A__		 =				[]
  A__		 =				[]





  def __init__(		self      :	int							,					_a      :	Tuple							,					_a      :	Optional[int]=None							,					_a      :	str=None							,					_a      :	Tuple="</s>"							,					_a      :	List[str]="</s>"							,					_a      :	Any="<s>"							,					_a      :	Dict="<unk>"							,					_a      :	Optional[Any]="<pad>"							,					_a      :	Optional[int]="<mask>"							,					_a      :	Optional[Dict[str, Any]] = None							,					**_a      :	List[Any]							,					)	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =AddedToken(_a							,					lstrip=_a							,					rstrip=_a ) if isinstance(_a							,					_a ) else mask_token

       _SCREAMING_SNAKE_CASE							    ={} if sp_model_kwargs is None else sp_model_kwargs

       _SCREAMING_SNAKE_CASE							    =kwargs.get('additional_special_tokens'							,					[] )
       kwargs["additional_special_tokens"] += [
           code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           src_lang=_a							,					tgt_lang=_a							,					eos_token=_a							,					unk_token=_a							,					sep_token=_a							,					cls_token=_a							,					pad_token=_a							,					mask_token=_a							,					sp_model_kwargs=self.sp_model_kwargs							,					**_a							,					)

       _SCREAMING_SNAKE_CASE							    =spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(str(_a ) )
       _SCREAMING_SNAKE_CASE							    =vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # Mimic fairseq token-to-id alignment for the first 4 token
       _SCREAMING_SNAKE_CASE							    ={'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       _SCREAMING_SNAKE_CASE							    =1

       _SCREAMING_SNAKE_CASE							    =len(self.sp_model )
       _SCREAMING_SNAKE_CASE							    ={
           code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_a )
       }
       _SCREAMING_SNAKE_CASE							    ={v: k for k, v in self.lang_code_to_id.items()}
       _SCREAMING_SNAKE_CASE							    =len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset

       self.fairseq_tokens_to_ids.update(self.lang_code_to_id )
       _SCREAMING_SNAKE_CASE							    ={v: k for k, v in self.fairseq_tokens_to_ids.items()}

       _SCREAMING_SNAKE_CASE							    =src_lang if src_lang is not None else 'en_XX'
       _SCREAMING_SNAKE_CASE							    =self.lang_code_to_id[self._src_lang]
       _SCREAMING_SNAKE_CASE							    =tgt_lang
       self.set_src_lang_special_tokens(self._src_lang )





  @property
  def 						A  (		self      :	Dict )	->    int:





       '''simple docstring'''






       return len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset + 1  # Plus 1 for the mask token





  @property
  def 						A  (		self      :	List[Any] )	->    str:





       '''simple docstring'''






       return self._src_lang





  @src_lang.setter
  def 						A  (		self      :	Optional[Any]							,					_a      :	str )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =new_src_lang
       self.set_src_lang_special_tokens(self._src_lang )





  def __getstate__(		self      :	List[Any] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.__dict__.copy()
       _SCREAMING_SNAKE_CASE							    =None
       return state





  def __setstate__(		self      :	Dict							,					_a      :	Dict )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =d

       # for backward compatibility
       if not hasattr(self							,					'sp_model_kwargs' ):
            _SCREAMING_SNAKE_CASE							    ={}

       _SCREAMING_SNAKE_CASE							    =spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(self.vocab_file )





  def 						A  (		self      :	List[Any] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ={self.convert_ids_to_tokens(_a ): i for i in range(self.vocab_size )}
       vocab.update(self.added_tokens_encoder )
       return vocab





  def 						A  (		self      :	Dict							,					_a      :	str )	->    List[str]:





       '''simple docstring'''






       return self.sp_model.encode(_a							,					out_type=_a )





  def 						A  (		self      :	str							,					_a      :	str )	->    int:





       '''simple docstring'''






       if token in self.fairseq_tokens_to_ids:
            return self.fairseq_tokens_to_ids[token]
       _SCREAMING_SNAKE_CASE							    =self.sp_model.PieceToId(_a )

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





  def 						A  (		self      :	Optional[int]							,					_a      :	int )	->    str:





       '''simple docstring'''






       if index in self.fairseq_ids_to_tokens:
            return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset )





  def 						A  (		self      :	List[Any]							,					_a      :	str )	->    int:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =[]
       _SCREAMING_SNAKE_CASE							    =''
       _SCREAMING_SNAKE_CASE							    =False
       for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                 if not prev_is_special:
                      out_string += " "
                 out_string += self.sp_model.decode(_a ) + token
                 _SCREAMING_SNAKE_CASE							    =True
                 _SCREAMING_SNAKE_CASE							    =[]
            else:
                 current_sub_tokens.append(_a )
                 _SCREAMING_SNAKE_CASE							    =False
       out_string += self.sp_model.decode(_a )
       return out_string.strip()





  def 						A  (		self      :	Optional[int]							,					_a      :	str							,					_a      :	Optional[str] = None )	->    Tuple[str]:





       '''simple docstring'''






       if not os.path.isdir(_a ):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory" )
            return
       _SCREAMING_SNAKE_CASE							    =os.path.join(
           _a							,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )

       if os.path.abspath(self.vocab_file ) != os.path.abspath(_a ) and os.path.isfile(self.vocab_file ):
            copyfile(self.vocab_file							,					_a )
       elif not os.path.isfile(self.vocab_file ):
            with open(_a							,					'wb' ) as fi:
                 _SCREAMING_SNAKE_CASE							    =self.sp_model.serialized_model_proto()
                 fi.write(_a )

       return (out_vocab_file,)





  def 						A  (		self      :	str							,					_a      :	List[int]							,					_a      :	Optional[List[int]] = None							,					_a      :	bool = False )	->    List[int]:





       '''simple docstring'''






       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=_a							,					token_ids_a=_a							,					already_has_special_tokens=_a )

       _SCREAMING_SNAKE_CASE							    =[1] * len(self.prefix_tokens )
       _SCREAMING_SNAKE_CASE							    =[1] * len(self.suffix_tokens )
       if token_ids_a is None:
            return prefix_ones + ([0] * len(_a )) + suffix_ones
       return prefix_ones + ([0] * len(_a )) + ([0] * len(_a )) + suffix_ones





  def 						A  (		self      :	List[str]							,					_a      :	List[int]							,					_a      :	Optional[List[int]] = None )	->    List[int]:





       '''simple docstring'''






       if token_ids_a is None:
            return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





  def 						A  (		self      :	Union[str, Any]							,					_a      :	Union[str, Any]							,					_a      :	str							,					_a      :	Optional[str]							,					_a      :	Optional[str]							,					**_a      :	Optional[Any] )	->    int:





       '''simple docstring'''






       if src_lang is None or tgt_lang is None:
            raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model' )
       _SCREAMING_SNAKE_CASE							    =src_lang
       _SCREAMING_SNAKE_CASE							    =self(_a							,					add_special_tokens=_a							,					return_tensors=_a							,					**_a )
       _SCREAMING_SNAKE_CASE							    =self.convert_tokens_to_ids(_a )
       _SCREAMING_SNAKE_CASE							    =tgt_lang_id
       return inputs





  def 						A  (		self      :	Optional[Any]							,					_a      :	List[str]							,					_a      :	str = "en_XX"							,					_a      :	Optional[List[str]] = None							,					_a      :	str = "ro_RO"							,					**_a      :	Optional[Any]							,					)	->    BatchEncoding:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =src_lang
       _SCREAMING_SNAKE_CASE							    =tgt_lang
       return super().prepare_seqaseq_batch(_a							,					_a							,					**_a )





  def 						A  (		self      :	int )	->    List[Any]:





       '''simple docstring'''






       return self.set_src_lang_special_tokens(self.src_lang )





  def 						A  (		self      :	Optional[Any] )	->    Optional[Any]:





       '''simple docstring'''






       return self.set_tgt_lang_special_tokens(self.tgt_lang )





  def 						A  (		self      :	Tuple							,					_a      :	str )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.lang_code_to_id[src_lang]
       _SCREAMING_SNAKE_CASE							    =[self.cur_lang_code_id]
       _SCREAMING_SNAKE_CASE							    =[self.eos_token_id]





  def 						A  (		self      :	Optional[int]							,					_a      :	str )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.lang_code_to_id[tgt_lang]
       _SCREAMING_SNAKE_CASE							    =[self.cur_lang_code_id]
       _SCREAMING_SNAKE_CASE							    =[self.eos_token_id]






import argparse
import os
import re

import packaging.version


UpperCamelCase__    			=			"""examples/"""
UpperCamelCase__    			=			{
    """examples""": (re.compile(R"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(R"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(R"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), R"""\1version=\"VERSION\","""),
    """doc""": (re.compile(R"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
UpperCamelCase__    			=			{
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
UpperCamelCase__    			=			"""README.md"""







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Optional[int]      ,      SCREAMING_SNAKE_CASE_							:				Any      ,      SCREAMING_SNAKE_CASE_							:				List[str]     ):
					with open(SCREAMING_SNAKE_CASE_      ,      "r"      ,      encoding="utf-8"      ,      newline="\n"     ) as f:
										__lowerCAmelCase				   =       f.read()
					__lowerCAmelCase   ,							__lowerCAmelCase				   =       REPLACE_PATTERNS[pattern]
					__lowerCAmelCase				   =       replace.replace("VERSION"      ,      SCREAMING_SNAKE_CASE_     )
					__lowerCAmelCase				   =       re_pattern.sub(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
					with open(SCREAMING_SNAKE_CASE_      ,      "w"      ,      encoding="utf-8"      ,      newline="\n"     ) as f:
										f.write(SCREAMING_SNAKE_CASE_     )







def 	_a      (						SCREAMING_SNAKE_CASE_							:				List[Any]     ):
					for folder, directories, fnames in os.walk(SCREAMING_SNAKE_CASE_     ):
										# Removing some of the folders with non-actively maintained examples from the walk
										if "research_projects" in directories:
															directories.remove("research_projects"     )
										if "legacy" in directories:
															directories.remove("legacy"     )
										for fname in fnames:
															if fname.endswith(".py"     ):
																				update_version_in_file(os.path.join(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )      ,      SCREAMING_SNAKE_CASE_      ,      pattern="examples"     )







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Tuple      ,      SCREAMING_SNAKE_CASE_							:				Optional[int]=False     ):
					for pattern, fname in REPLACE_FILES.items():
										update_version_in_file(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
					if not patch:
										update_version_in_examples(SCREAMING_SNAKE_CASE_     )







def 	_a      (						):
					__lowerCAmelCase				   =       "🤗 Transformers currently provides the following architectures"
					__lowerCAmelCase				   =       "1. Want to contribute a new model?"
					with open(SCREAMING_SNAKE_CASE_      ,      "r"      ,      encoding="utf-8"      ,      newline="\n"     ) as f:
										__lowerCAmelCase				   =       f.readlines()

					# Find the start of the list.
					__lowerCAmelCase				   =       0
					while not lines[start_index].startswith(_start_prompt     ):
										start_index += 1
					start_index += 1

					__lowerCAmelCase				   =       start_index
					# Update the lines in the model list.
					while not lines[index].startswith(_end_prompt     ):
										if lines[index].startswith("1."     ):
															__lowerCAmelCase				   =       lines[index].replace(
															    "https://huggingface.co/docs/transformers/main/model_doc"      ,      "https://huggingface.co/docs/transformers/model_doc"      ,      )
										index += 1

					with open(SCREAMING_SNAKE_CASE_      ,      "w"      ,      encoding="utf-8"      ,      newline="\n"     ) as f:
										f.writelines(SCREAMING_SNAKE_CASE_     )







def 	_a      (						):
					with open(REPLACE_FILES["init"]      ,      "r"     ) as f:
										__lowerCAmelCase				   =       f.read()
					__lowerCAmelCase				   =       REPLACE_PATTERNS["init"][0].search(SCREAMING_SNAKE_CASE_     ).groups()[0]
					return packaging.version.parse(SCREAMING_SNAKE_CASE_     )







def 	_a      (						SCREAMING_SNAKE_CASE_							:				List[Any]=False     ):
					__lowerCAmelCase				   =       get_version()
					if patch and default_version.is_devrelease:
										raise ValueError("Can't create a patch version from the dev branch, checkout a released version!"     )
					if default_version.is_devrelease:
										__lowerCAmelCase				   =       default_version.base_version
					elif patch:
										__lowerCAmelCase				   =       F"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
					else:
										__lowerCAmelCase				   =       F"""{default_version.major}.{default_version.minor + 1}.0"""

					# Now let's ask nicely if that's the right one.
					__lowerCAmelCase				   =       input(F"""Which version are you releasing? [{default_version}]"""     )
					if len(SCREAMING_SNAKE_CASE_     ) == 0:
										__lowerCAmelCase				   =       default_version

					print(F"""Updating version to {version}."""     )
					global_version_update(SCREAMING_SNAKE_CASE_      ,      patch=SCREAMING_SNAKE_CASE_     )
					if not patch:
										print("Cleaning main README, don't forget to run `make fix-copies`."     )
										clean_main_ref_in_model_list()







def 	_a      (						):
					__lowerCAmelCase				   =       get_version()
					__lowerCAmelCase				   =       F"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
					__lowerCAmelCase				   =       current_version.base_version

					# Check with the user we got that right.
					__lowerCAmelCase				   =       input(F"""Which version are we developing now? [{dev_version}]"""     )
					if len(SCREAMING_SNAKE_CASE_     ) == 0:
										__lowerCAmelCase				   =       dev_version

					print(F"""Updating version to {version}."""     )
					global_version_update(SCREAMING_SNAKE_CASE_     )
					print("Cleaning main README, don't forget to run `make fix-copies`."     )
					clean_main_ref_in_model_list()


if __name__ == "__main__":
					UpperCamelCase__    			=			argparse.ArgumentParser()
					parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
					parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
					UpperCamelCase__    			=			parser.parse_args()
					if not args.post_release:
										pre_release_work(patch=args.patch)
					elif args.patch:
										print("""Nothing to do after a patch :-)""")
					else:
										post_release_work()




