




import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase		:				Any       			=		logging.get_logger(__name__)

__lowerCAmelCase		:				int       			=		'▁'

__lowerCAmelCase		:				Union[str, Any]       			=		{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__lowerCAmelCase		:				int       			=		{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

__lowerCAmelCase		:				List[str]       			=		{
    'facebook/m2m100_418M': 1024,
}

# fmt: off
__lowerCAmelCase		:				List[Any]       			=		{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}




class     snake_case__							(_UpperCamelCase   ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_       :							Any							 =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_       :							List[str]							 =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_       :							Any							 =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_       :							Union[str, Any]							 =					["""input_ids""", """attention_mask"""]

				SCREAMING_SNAKE_CASE_       :							List[int]							 =					[]
				SCREAMING_SNAKE_CASE_       :							List[int]							 =					[]





				def __init__(      self				:     Union[str, Any]					,      __lowerCamelCase				:     Dict					,      __lowerCamelCase				:     Union[str, Any]					,      __lowerCamelCase				:     Tuple=None					,      __lowerCamelCase				:     List[str]=None					,      __lowerCamelCase				:     List[str]="<s>"					,      __lowerCamelCase				:     List[Any]="</s>"					,      __lowerCamelCase				:     Tuple="</s>"					,      __lowerCamelCase				:     Union[str, Any]="<pad>"					,      __lowerCamelCase				:     str="<unk>"					,      __lowerCamelCase				:     int="m2m100"					,      __lowerCamelCase				:     Optional[Dict[str, Any]] = None					,      __lowerCamelCase				:     int=8					,      **__lowerCamelCase				:     Dict					,      )					->     None:
							a    =			{} if sp_model_kwargs is None else sp_model_kwargs

							a    =			language_codes
							a    =			FAIRSEQ_LANGUAGE_CODES[language_codes]
							a    =			{lang_code: f"""__{lang_code}__""" for lang_code in fairseq_language_code}

							a    =			kwargs.get("additional_special_tokens"					,      []							)
							kwargs["additional_special_tokens"] += [
							    self.get_lang_token(__lowerCamelCase							)
							    for lang_code in fairseq_language_code
							    if self.get_lang_token(__lowerCamelCase							) not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    src_lang=__lowerCamelCase					,      tgt_lang=__lowerCamelCase					,      bos_token=__lowerCamelCase					,      eos_token=__lowerCamelCase					,      sep_token=__lowerCamelCase					,      unk_token=__lowerCamelCase					,      pad_token=__lowerCamelCase					,      language_codes=__lowerCamelCase					,      sp_model_kwargs=self.sp_model_kwargs					,      num_madeup_words=__lowerCamelCase					,      **__lowerCamelCase					,      )

							a    =			vocab_file
							a    =			load_json(__lowerCamelCase							)
							a    =			{v: k for k, v in self.encoder.items()}
							a    =			spm_file
							a    =			load_spm(__lowerCamelCase					,      self.sp_model_kwargs							)

							a    =			len(self.encoder							)

							a    =			{
							    self.get_lang_token(__lowerCamelCase							): self.encoder_size + i for i, lang_code in enumerate(__lowerCamelCase							)
							}
							a    =			{lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowerCamelCase							)}
							a    =			{v: k for k, v in self.lang_token_to_id.items()}

							a    =			src_lang if src_lang is not None else "en"
							a    =			tgt_lang
							a    =			self.get_lang_id(self._src_lang							)
							self.set_src_lang_special_tokens(self._src_lang							)

							a    =			num_madeup_words





				@property
				def 							__UpperCAmelCase							(      self				:     str							)					->     int:
							return len(self.encoder							) + len(self.lang_token_to_id							)





				@property
				def 							__UpperCAmelCase							(      self				:     Optional[Any]							)					->     str:
							return self._src_lang





				@src_lang.setter
				def 							__UpperCAmelCase							(      self				:     Tuple					,      __lowerCamelCase				:     str							)					->     None:
							a    =			new_src_lang
							self.set_src_lang_special_tokens(self._src_lang							)





				def 							__UpperCAmelCase							(      self				:     Tuple					,      __lowerCamelCase				:     str							)					->     List[str]:
							return self.sp_model.encode(__lowerCamelCase					,      out_type=__lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     Dict					,      __lowerCamelCase				:     Any							)					->     Tuple:
							if token in self.lang_token_to_id:
										return self.lang_token_to_id[token]
							return self.encoder.get(__lowerCamelCase					,      self.encoder[self.unk_token]							)





				def 							__UpperCAmelCase							(      self				:     Optional[Any]					,      __lowerCamelCase				:     int							)					->     str:
							if index in self.id_to_lang_token:
										return self.id_to_lang_token[index]
							return self.decoder.get(__lowerCamelCase					,      self.unk_token							)





				def 							__UpperCAmelCase							(      self				:     List[str]					,      __lowerCamelCase				:     List[str]							)					->     List[str]:
							a    =			[]
							a    =			""
							for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
													out_string += self.sp_model.decode(__lowerCamelCase							) + token
													a    =			[]
										else:
													current_sub_tokens.append(__lowerCamelCase							)
							out_string += self.sp_model.decode(__lowerCamelCase							)
							return out_string.strip()





				def 							__UpperCAmelCase							(      self				:     Optional[Any]					,      __lowerCamelCase				:     List[int]					,      __lowerCamelCase				:     Optional[List[int]] = None					,      __lowerCamelCase				:     bool = False							)					->     List[int]:

							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=__lowerCamelCase					,      token_ids_a=__lowerCamelCase					,      already_has_special_tokens=__lowerCamelCase							)

							a    =			[1] * len(self.prefix_tokens							)
							a    =			[1] * len(self.suffix_tokens							)
							if token_ids_a is None:
										return prefix_ones + ([0] * len(__lowerCamelCase							)) + suffix_ones
							return prefix_ones + ([0] * len(__lowerCamelCase							)) + ([0] * len(__lowerCamelCase							)) + suffix_ones





				def 							__UpperCAmelCase							(      self				:     int					,      __lowerCamelCase				:     List[int]					,      __lowerCamelCase				:     Optional[List[int]] = None							)					->     List[int]:
							if token_ids_a is None:
										return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





				def 							__UpperCAmelCase							(      self				:     Any							)					->     Dict:
							a    =			{self.convert_ids_to_tokens(__lowerCamelCase							): i for i in range(self.vocab_size							)}
							vocab.update(self.added_tokens_encoder							)
							return vocab





				def __getstate__(      self				:     Tuple							)					->     Dict:
							a    =			self.__dict__.copy()
							a    =			None
							return state





				def __setstate__(      self				:     Tuple					,      __lowerCamelCase				:     Dict							)					->     None:
							a    =			d

							# for backward compatibility
							if not hasattr(self					,      "sp_model_kwargs"							):
										a    =			{}

							a    =			load_spm(self.spm_file					,      self.sp_model_kwargs							)





				def 							__UpperCAmelCase							(      self				:     Tuple					,      __lowerCamelCase				:     str					,      __lowerCamelCase				:     Optional[str] = None							)					->     Tuple[str]:
							a    =			Path(__lowerCamelCase							)
							if not save_dir.is_dir():
										raise OSError(f"""{save_directory} should be a directory"""							)
							a    =			save_dir / (
							    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
							)
							a    =			save_dir / (
							    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
							)

							save_json(self.encoder					,      __lowerCamelCase							)

							if os.path.abspath(self.spm_file							) != os.path.abspath(__lowerCamelCase							) and os.path.isfile(self.spm_file							):
										copyfile(self.spm_file					,      __lowerCamelCase							)
							elif not os.path.isfile(self.spm_file							):
										with open(__lowerCamelCase					,      "wb"							) as fi:
													a    =			self.sp_model.serialized_model_proto()
													fi.write(__lowerCamelCase							)

							return (str(__lowerCamelCase							), str(__lowerCamelCase							))





				def 							__UpperCAmelCase							(      self				:     Dict					,      __lowerCamelCase				:     List[str]					,      __lowerCamelCase				:     str = "en"					,      __lowerCamelCase				:     Optional[List[str]] = None					,      __lowerCamelCase				:     str = "ro"					,      **__lowerCamelCase				:     List[Any]					,      )					->     BatchEncoding:
							a    =			src_lang
							a    =			tgt_lang
							self.set_src_lang_special_tokens(self.src_lang							)
							return super().prepare_seqaseq_batch(__lowerCamelCase					,      __lowerCamelCase					,      **__lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     int					,      __lowerCamelCase				:     List[str]					,      __lowerCamelCase				:     Optional[str]					,      __lowerCamelCase				:     Optional[str]					,      **__lowerCamelCase				:     Optional[int]							)					->     List[Any]:
							if src_lang is None or tgt_lang is None:
										raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"							)
							a    =			src_lang
							a    =			self(__lowerCamelCase					,      add_special_tokens=__lowerCamelCase					,      **__lowerCamelCase							)
							a    =			self.get_lang_id(__lowerCamelCase							)
							a    =			tgt_lang_id
							return inputs





				def 							__UpperCAmelCase							(      self				:     Optional[int]							)					->     int:
							self.set_src_lang_special_tokens(self.src_lang							)





				def 							__UpperCAmelCase							(      self				:     Optional[Any]							)					->     int:
							self.set_tgt_lang_special_tokens(self.tgt_lang							)





				def 							__UpperCAmelCase							(      self				:     List[Any]					,      __lowerCamelCase				:     str							)					->     None:
							a    =			self.get_lang_token(__lowerCamelCase							)
							a    =			self.lang_token_to_id[lang_token]
							a    =			[self.cur_lang_id]
							a    =			[self.eos_token_id]





				def 							__UpperCAmelCase							(      self				:     int					,      __lowerCamelCase				:     str							)					->     None:
							a    =			self.get_lang_token(__lowerCamelCase							)
							a    =			self.lang_token_to_id[lang_token]
							a    =			[self.cur_lang_id]
							a    =			[self.eos_token_id]





				def 							__UpperCAmelCase							(      self				:     List[str]					,      __lowerCamelCase				:     str							)					->     str:
							return self.lang_code_to_token[lang]





				def 							__UpperCAmelCase							(      self				:     Optional[int]					,      __lowerCamelCase				:     str							)					->     int:
							a    =			self.get_lang_token(__lowerCamelCase							)
							return self.lang_token_to_id[lang_token]




def 			__magic_name__				( A		:					str,  A		:					Dict[str, Any]   ):






			'''simple docstring'''







			a    =			sentencepiece.SentencePieceProcessor(**A   )
			spm.Load(str(A   )   )
			return spm




def 			__magic_name__				( A		:					str   ):






			'''simple docstring'''







			with open(A,  "r"   ) as f:
						return json.load(A   )






def 			__magic_name__				( A		:					Union[str, Any],  A		:					str   ):






			'''simple docstring'''







			with open(A,  "w"   ) as f:
						json.dump(A,  A,  indent=2   )












from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase		:				Optional[Any]       			=		logging.get_logger(__name__)

__lowerCAmelCase		:				str       			=		{'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/config.json'}




class     snake_case__							(_UpperCamelCase   ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_       :							Dict							 =					"""openai-gpt"""
				SCREAMING_SNAKE_CASE_       :							Tuple							 =					{
				    """max_position_embeddings""": """n_positions""",
				    """hidden_size""": """n_embd""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}





				def __init__(      self				:     str					,      __lowerCamelCase				:     List[str]=4_04_78					,      __lowerCamelCase				:     List[Any]=5_12					,      __lowerCamelCase				:     List[str]=7_68					,      __lowerCamelCase				:     List[str]=12					,      __lowerCamelCase				:     Optional[Any]=12					,      __lowerCamelCase				:     str="gelu"					,      __lowerCamelCase				:     Optional[int]=0.1					,      __lowerCamelCase				:     Any=0.1					,      __lowerCamelCase				:     Dict=0.1					,      __lowerCamelCase				:     Any=1e-5					,      __lowerCamelCase				:     Optional[int]=0.02					,      __lowerCamelCase				:     Optional[int]="cls_index"					,      __lowerCamelCase				:     List[Any]=True					,      __lowerCamelCase				:     Optional[int]=None					,      __lowerCamelCase				:     Optional[Any]=True					,      __lowerCamelCase				:     Tuple=0.1					,      **__lowerCamelCase				:     Union[str, Any]					,      )					->     List[str]:
							a    =			vocab_size
							a    =			n_positions
							a    =			n_embd
							a    =			n_layer
							a    =			n_head
							a    =			afn
							a    =			resid_pdrop
							a    =			embd_pdrop
							a    =			attn_pdrop
							a    =			layer_norm_epsilon
							a    =			initializer_range
							a    =			summary_type
							a    =			summary_use_proj
							a    =			summary_activation
							a    =			summary_first_dropout
							a    =			summary_proj_to_labels
							super().__init__(**__lowerCamelCase							)






