




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_nllb import NllbTokenizer
else:
       UpperCAmelCase_									=	None


UpperCAmelCase_									=	logging.get_logger(__name__)


UpperCAmelCase_									=	{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_									=	{
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_									=	{
    'facebook/nllb-large-en-ro': 1_024,
    'facebook/nllb-200-distilled-600M': 1_024,
}

# fmt: off
UpperCAmelCase_									=	['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']





class     lowerCamelCase__(					__lowerCamelCase):
      UpperCAmelCase__      :	Optional[int]    		=   VOCAB_FILES_NAMES
      UpperCAmelCase__      :	Tuple    		=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__      :	int    		=   PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__      :	str    		=   ['input_ids', 'attention_mask']
      UpperCAmelCase__      :	str    		=   NllbTokenizer

      UpperCAmelCase__      :	List[int]    		=   []
      UpperCAmelCase__      :	List[int]    		=   []
      def __init__(				self: Dict    ,       UpperCamelCase_: Union[str, Any]=None    ,       UpperCamelCase_: Optional[int]=None    ,       UpperCamelCase_: Any="<s>"    ,       UpperCamelCase_: Any="</s>"    ,       UpperCamelCase_: List[Any]="</s>"    ,       UpperCamelCase_: Optional[Any]="<s>"    ,       UpperCamelCase_: int="<unk>"    ,       UpperCamelCase_: Union[str, Any]="<pad>"    ,       UpperCamelCase_: Union[str, Any]="<mask>"    ,       UpperCamelCase_: Union[str, Any]=None    ,       UpperCamelCase_: str=None    ,       UpperCamelCase_: Dict=None    ,       UpperCamelCase_: Optional[Any]=False    ,       **UpperCamelCase_: int    ,       ):
             # Mask token behave like a normal word, i.e. include the space before it
             __lowerCamelCase   =						AddedToken(UpperCamelCase_    ,       lstrip=UpperCamelCase_    ,       rstrip=UpperCamelCase_   ) if isinstance(UpperCamelCase_    ,       UpperCamelCase_   ) else mask_token
             __lowerCamelCase   =						legacy_behaviour
             super().__init__(
                 vocab_file=UpperCamelCase_    ,       tokenizer_file=UpperCamelCase_    ,       bos_token=UpperCamelCase_    ,       eos_token=UpperCamelCase_    ,       sep_token=UpperCamelCase_    ,       cls_token=UpperCamelCase_    ,       unk_token=UpperCamelCase_    ,       pad_token=UpperCamelCase_    ,       mask_token=UpperCamelCase_    ,       src_lang=UpperCamelCase_    ,       tgt_lang=UpperCamelCase_    ,       additional_special_tokens=UpperCamelCase_    ,       legacy_behaviour=UpperCamelCase_    ,       **UpperCamelCase_    ,       )

             __lowerCamelCase   =						vocab_file
             __lowerCamelCase   =						False if not self.vocab_file else True

             __lowerCamelCase   =						FAIRSEQ_LANGUAGE_CODES.copy()

             if additional_special_tokens is not None:
                    # Only add those special tokens if they are not already there.
                    _additional_special_tokens.extend(
                        [t for t in additional_special_tokens if t not in _additional_special_tokens]   )

             self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}   )
             __lowerCamelCase   =						{
                 lang_code: self.convert_tokens_to_ids(UpperCamelCase_   ) for lang_code in FAIRSEQ_LANGUAGE_CODES
             }

             __lowerCamelCase   =						src_lang if src_lang is not None else """eng_Latn"""
             __lowerCamelCase   =						self.convert_tokens_to_ids(self._src_lang   )
             __lowerCamelCase   =						tgt_lang
             self.set_src_lang_special_tokens(self._src_lang   )
      @property
      def 			lowerCAmelCase__						(				self: int   ):
             return self._src_lang
      @src_lang.setter
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: str   ):
             __lowerCamelCase   =						new_src_lang
             self.set_src_lang_special_tokens(self._src_lang   )
      def 			lowerCAmelCase__						(				self: int    ,       UpperCamelCase_: List[int]    ,       UpperCamelCase_: Optional[List[int]] = None   ):
             if token_ids_a is None:
                    return self.prefix_tokens + token_ids_a + self.suffix_tokens
             # We don't expect to process pairs, but leave the pair logic for API consistency
             return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
      def 			lowerCAmelCase__						(				self: Any    ,       UpperCamelCase_: List[int]    ,       UpperCamelCase_: Optional[List[int]] = None   ):
             __lowerCamelCase   =						[self.sep_token_id]
             __lowerCamelCase   =						[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep   ) * [0]
             return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]
      def 			lowerCAmelCase__						(				self: Tuple    ,       UpperCamelCase_: str    ,       UpperCamelCase_: str    ,       UpperCamelCase_: Optional[str]    ,       UpperCamelCase_: Optional[str]    ,       **UpperCamelCase_: int   ):
             if src_lang is None or tgt_lang is None:
                    raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""   )
             __lowerCamelCase   =						src_lang
             __lowerCamelCase   =						self(UpperCamelCase_    ,       add_special_tokens=UpperCamelCase_    ,       return_tensors=UpperCamelCase_    ,       **UpperCamelCase_   )
             __lowerCamelCase   =						self.convert_tokens_to_ids(UpperCamelCase_   )
             __lowerCamelCase   =						tgt_lang_id
             return inputs
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: List[str]    ,       UpperCamelCase_: str = "eng_Latn"    ,       UpperCamelCase_: Optional[List[str]] = None    ,       UpperCamelCase_: str = "fra_Latn"    ,       **UpperCamelCase_: Optional[int]    ,       ):
             __lowerCamelCase   =						src_lang
             __lowerCamelCase   =						tgt_lang
             return super().prepare_seqaseq_batch(UpperCamelCase_    ,       UpperCamelCase_    ,       **UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Dict   ):
             return self.set_src_lang_special_tokens(self.src_lang   )
      def 			lowerCAmelCase__						(				self: List[str]   ):
             return self.set_tgt_lang_special_tokens(self.tgt_lang   )
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: Optional[int]   ):
             __lowerCamelCase   =						self.convert_tokens_to_ids(UpperCamelCase_   )

             if self.legacy_behaviour:
                    __lowerCamelCase   =						[]
                    __lowerCamelCase   =						[self.eos_token_id, self.cur_lang_code]
             else:
                    __lowerCamelCase   =						[self.cur_lang_code]
                    __lowerCamelCase   =						[self.eos_token_id]

             __lowerCamelCase   =						self.convert_ids_to_tokens(self.prefix_tokens   )
             __lowerCamelCase   =						self.convert_ids_to_tokens(self.suffix_tokens   )

             __lowerCamelCase   =						processors.TemplateProcessing(
                 single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,       self.prefix_tokens + self.suffix_tokens   )   )    ,       )
      def 			lowerCAmelCase__						(				self: Any    ,       UpperCamelCase_: str   ):
             __lowerCamelCase   =						self.convert_tokens_to_ids(UpperCamelCase_   )
             if self.legacy_behaviour:
                    __lowerCamelCase   =						[]
                    __lowerCamelCase   =						[self.eos_token_id, self.cur_lang_code]
             else:
                    __lowerCamelCase   =						[self.cur_lang_code]
                    __lowerCamelCase   =						[self.eos_token_id]

             __lowerCamelCase   =						self.convert_ids_to_tokens(self.prefix_tokens   )
             __lowerCamelCase   =						self.convert_ids_to_tokens(self.suffix_tokens   )

             __lowerCamelCase   =						processors.TemplateProcessing(
                 single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,       self.prefix_tokens + self.suffix_tokens   )   )    ,       )




      def 			lowerCAmelCase__						(				self: Any    ,       UpperCamelCase_: str    ,       UpperCamelCase_: Optional[str] = None   ):
             if not self.can_save_slow_tokenizer:
                    raise ValueError(
                        """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                        """tokenizer."""   )

             if not os.path.isdir(UpperCamelCase_   ):
                    logger.error(F'Vocabulary path ({save_directory}) should be a directory.'   )
                    return
             __lowerCamelCase   =						os.path.join(
                 UpperCamelCase_    ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

             if os.path.abspath(self.vocab_file   ) != os.path.abspath(UpperCamelCase_   ):
                    copyfile(self.vocab_file    ,       UpperCamelCase_   )

             return (out_vocab_file,)









"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_   : Optional[int]   			= logging.get_logger(__name__)

UpperCAmelCase_   : Any   			= {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/config.json"""}




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  "openai-gpt"
      __UpperCamelCase	 =  {
          "max_position_embeddings": "n_positions",
          "hidden_size": "n_embd",
          "num_attention_heads": "n_head",
          "num_hidden_layers": "n_layer",
      }
      def __init__(						self     :  List[str]						,	lowercase_     :  List[str]=40478						,	lowercase_     :  List[str]=512						,	lowercase_     :  Optional[Any]=768						,	lowercase_     :  Tuple=12						,	lowercase_     :  Tuple=12						,	lowercase_     :  Union[str, Any]="gelu"						,	lowercase_     :  List[Any]=0.1						,	lowercase_     :  Tuple=0.1						,	lowercase_     :  List[Any]=0.1						,	lowercase_     :  List[Any]=1e-5						,	lowercase_     :  int=0.02						,	lowercase_     :  Optional[int]="cls_index"						,	lowercase_     :  Any=True						,	lowercase_     :  List[Any]=None						,	lowercase_     :  List[str]=True						,	lowercase_     :  Optional[Any]=0.1						,	**lowercase_     :  List[str]						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		vocab_size
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		n_positions
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		n_embd
          SCREAMING_SNAKE_CASE_      :  Dict    						=		n_layer
          SCREAMING_SNAKE_CASE_      :  Any    						=		n_head
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		afn
          SCREAMING_SNAKE_CASE_      :  int    						=		resid_pdrop
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		embd_pdrop
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		attn_pdrop
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		layer_norm_epsilon
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		initializer_range
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		summary_type
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		summary_use_proj
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		summary_activation
          SCREAMING_SNAKE_CASE_      :  Any    						=		summary_first_dropout
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		summary_proj_to_labels
          super().__init__(**lowercase_)


