






"""simple docstring"""






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_   : Any   			= logging.get_logger(__name__)

UpperCAmelCase_   : List[str]   			= """▁"""

UpperCAmelCase_   : Tuple   			= {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

UpperCAmelCase_   : Optional[Any]   			= {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

UpperCAmelCase_   : Tuple   			= {
    """facebook/m2m100_418M""": 1024,
}

# fmt: off
UpperCAmelCase_   : Optional[Any]   			= {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  ["input_ids", "attention_mask"]

      __UpperCamelCase	 =  []
      __UpperCamelCase	 =  []
      def __init__(						self     :  List[str]						,	lowercase_     :  Tuple						,	lowercase_     :  List[str]						,	lowercase_     :  Optional[int]=None						,	lowercase_     :  Optional[int]=None						,	lowercase_     :  Union[str, Any]="<s>"						,	lowercase_     :  Optional[int]="</s>"						,	lowercase_     :  Any="</s>"						,	lowercase_     :  Optional[int]="<pad>"						,	lowercase_     :  List[str]="<unk>"						,	lowercase_     :  List[Any]="m2m100"						,	lowercase_     :  Optional[Dict[str, Any]] = None						,	lowercase_     :  Any=8						,	**lowercase_     :  List[str]						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  int    						=		{} if sp_model_kwargs is None else sp_model_kwargs

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		language_codes
          SCREAMING_SNAKE_CASE_      :  int    						=		FAIRSEQ_LANGUAGE_CODES[language_codes]
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		{lang_code: F'__{lang_code}__' for lang_code in fairseq_language_code}

          SCREAMING_SNAKE_CASE_      :  Tuple    						=		kwargs.get('''additional_special_tokens'''						,	[])
          kwargs["additional_special_tokens"] += [
              self.get_lang_token(lowercase_)
              for lang_code in fairseq_language_code
              if self.get_lang_token(lowercase_) not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              src_lang=lowercase_						,	tgt_lang=lowercase_						,	bos_token=lowercase_						,	eos_token=lowercase_						,	sep_token=lowercase_						,	unk_token=lowercase_						,	pad_token=lowercase_						,	language_codes=lowercase_						,	sp_model_kwargs=self.sp_model_kwargs						,	num_madeup_words=lowercase_						,	**lowercase_						,	)

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		vocab_file
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		load_json(lowercase_)
          SCREAMING_SNAKE_CASE_      :  Dict    						=		{v: k for k, v in self.encoder.items()}
          SCREAMING_SNAKE_CASE_      :  int    						=		spm_file
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		load_spm(lowercase_						,	self.sp_model_kwargs)

          SCREAMING_SNAKE_CASE_      :  Dict    						=		len(self.encoder)

          SCREAMING_SNAKE_CASE_      :  List[str]    						=		{
              self.get_lang_token(lowercase_): self.encoder_size + i for i, lang_code in enumerate(lowercase_)
          }
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		{lang_code: self.encoder_size + i for i, lang_code in enumerate(lowercase_)}
          SCREAMING_SNAKE_CASE_      :  str    						=		{v: k for k, v in self.lang_token_to_id.items()}

          SCREAMING_SNAKE_CASE_      :  str    						=		src_lang if src_lang is not None else '''en'''
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		tgt_lang
          SCREAMING_SNAKE_CASE_      :  Any    						=		self.get_lang_id(self._src_lang)
          self.set_src_lang_special_tokens(self._src_lang)

          SCREAMING_SNAKE_CASE_      :  Tuple    						=		num_madeup_words
      @property
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          return len(self.encoder) + len(self.lang_token_to_id)
      @property
      def _SCREAMING_SNAKE_CASE (						self     :  Any):




          '''simple docstring'''


          return self._src_lang
      @src_lang.setter
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]						,	lowercase_     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  int    						=		new_src_lang
          self.set_src_lang_special_tokens(self._src_lang)
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  str):




          '''simple docstring'''


          return self.sp_model.encode(lowercase_						,	out_type=lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Union[str, Any]						,	lowercase_     :  int):




          '''simple docstring'''


          if token in self.lang_token_to_id:
              return self.lang_token_to_id[token]
          return self.encoder.get(lowercase_						,	self.encoder[self.unk_token])
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]						,	lowercase_     :  int):




          '''simple docstring'''


          if index in self.id_to_lang_token:
              return self.id_to_lang_token[index]
          return self.decoder.get(lowercase_						,	self.unk_token)
      def _SCREAMING_SNAKE_CASE (						self     :  Any						,	lowercase_     :  List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		[]
          SCREAMING_SNAKE_CASE_      :  Dict    						=		''''''
          for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                  out_string += self.sp_model.decode(lowercase_) + token
                  SCREAMING_SNAKE_CASE_      :  str    						=		[]
              else:
                  current_sub_tokens.append(lowercase_)
          out_string += self.sp_model.decode(lowercase_)
          return out_string.strip()
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None						,	lowercase_     :  bool = False):




          '''simple docstring'''


          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=lowercase_						,	token_ids_a=lowercase_						,	already_has_special_tokens=lowercase_)

          SCREAMING_SNAKE_CASE_      :  Dict    						=		[1] * len(self.prefix_tokens)
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		[1] * len(self.suffix_tokens)
          if token_ids_a is None:
              return prefix_ones + ([0] * len(lowercase_)) + suffix_ones
          return prefix_ones + ([0] * len(lowercase_)) + ([0] * len(lowercase_)) + suffix_ones
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None):




          '''simple docstring'''


          if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		{self.convert_ids_to_tokens(lowercase_): i for i in range(self.vocab_size)}
          vocab.update(self.added_tokens_encoder)
          return vocab
      def __getstate__(						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		self.__dict__.copy()
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		None
          return state
      def __setstate__(						self     :  Optional[int]						,	lowercase_     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		d

          # for backward compatibility
          if not hasattr(self						,	'''sp_model_kwargs'''):
              SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		{}

          SCREAMING_SNAKE_CASE_      :  str    						=		load_spm(self.spm_file						,	self.sp_model_kwargs)
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		Path(lowercase_)
          if not save_dir.is_dir():
              raise OSError(F'{save_directory} should be a directory')
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		save_dir / (
              (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
          )
          SCREAMING_SNAKE_CASE_      :  int    						=		save_dir / (
              (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
          )

          save_json(self.encoder						,	lowercase_)

          if os.path.abspath(self.spm_file) != os.path.abspath(lowercase_) and os.path.isfile(self.spm_file):
              copyfile(self.spm_file						,	lowercase_)
          elif not os.path.isfile(self.spm_file):
              with open(lowercase_						,	'''wb''') as fi:
                  SCREAMING_SNAKE_CASE_      :  List[Any]    						=		self.sp_model.serialized_model_proto()
                  fi.write(lowercase_)

          return (str(lowercase_), str(lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  List[str]						,	lowercase_     :  str = "en"						,	lowercase_     :  Optional[List[str]] = None						,	lowercase_     :  str = "ro"						,	**lowercase_     :  List[Any]						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		src_lang
          SCREAMING_SNAKE_CASE_      :  int    						=		tgt_lang
          self.set_src_lang_special_tokens(self.src_lang)
          return super().prepare_seqaseq_batch(lowercase_						,	lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Tuple						,	lowercase_     :  Dict						,	lowercase_     :  Optional[str]						,	lowercase_     :  Optional[str]						,	**lowercase_     :  Optional[int]):




          '''simple docstring'''


          if src_lang is None or tgt_lang is None:
              raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model''')
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		src_lang
          SCREAMING_SNAKE_CASE_      :  str    						=		self(lowercase_						,	add_special_tokens=lowercase_						,	**lowercase_)
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		self.get_lang_id(lowercase_)
          SCREAMING_SNAKE_CASE_      :  Any    						=		tgt_lang_id
          return inputs
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          self.set_src_lang_special_tokens(self.src_lang)
      def _SCREAMING_SNAKE_CASE (						self     :  Tuple):




          '''simple docstring'''


          self.set_tgt_lang_special_tokens(self.tgt_lang)
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]						,	lowercase_     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		self.get_lang_token(lowercase_)
          SCREAMING_SNAKE_CASE_      :  str    						=		self.lang_token_to_id[lang_token]
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		[self.cur_lang_id]
          SCREAMING_SNAKE_CASE_      :  Any    						=		[self.eos_token_id]
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]						,	lowercase_     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		self.get_lang_token(lowercase_)
          SCREAMING_SNAKE_CASE_      :  Dict    						=		self.lang_token_to_id[lang_token]
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		[self.cur_lang_id]
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		[self.eos_token_id]
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]						,	lowercase_     :  str):




          '''simple docstring'''


          return self.lang_code_to_token[lang]



      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		self.get_lang_token(lowercase_)
          return self.lang_token_to_id[lang_token]



def 					_A			(__a    ,				__a							) ->       sentencepiece.SentencePieceProcessor:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_      :  Tuple    						=		sentencepiece.SentencePieceProcessor(**__a							)
    spm.Load(str(__a							)							)
    return spm



def 					_A			(__a							) ->       Union[Dict, List]:
    """simple docstring"""






    with open(__a    ,				'''r'''							) as f:
        return json.load(__a							)





def 					_A			(__a    ,				__a							) ->       None:
    """simple docstring"""






    with open(__a    ,				'''w'''							) as f:
        json.dump(__a    ,				__a    ,				indent=2							)










"""simple docstring"""






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


UpperCAmelCase_   : Optional[int]   			= logging.get_logger(__name__)

UpperCAmelCase_   : Dict   			= {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_   : Dict   			= {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

UpperCAmelCase_   : List[str]   			= {
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  ["input_ids", "attention_mask"]
      __UpperCamelCase	 =  GPTaTokenizer
      def __init__(						self     :  Optional[int]						,	lowercase_     :  int=None						,	lowercase_     :  List[str]=None						,	lowercase_     :  Union[str, Any]=None						,	lowercase_     :  Tuple="<|endoftext|>"						,	lowercase_     :  str="<|endoftext|>"						,	lowercase_     :  Dict="<|endoftext|>"						,	lowercase_     :  Tuple=False						,	**lowercase_     :  Optional[int]						,	):




          '''simple docstring'''


          super().__init__(
              lowercase_						,	lowercase_						,	tokenizer_file=lowercase_						,	unk_token=lowercase_						,	bos_token=lowercase_						,	eos_token=lowercase_						,	add_prefix_space=lowercase_						,	**lowercase_						,	)

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		kwargs.pop('''add_bos_token'''						,	lowercase_)

          SCREAMING_SNAKE_CASE_      :  Dict    						=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
          if pre_tok_state.get('''add_prefix_space'''						,	lowercase_) != add_prefix_space:
              SCREAMING_SNAKE_CASE_      :  int    						=		getattr(lowercase_						,	pre_tok_state.pop('''type'''))
              SCREAMING_SNAKE_CASE_      :  str    						=		add_prefix_space
              SCREAMING_SNAKE_CASE_      :  Dict    						=		pre_tok_class(**lowercase_)

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		add_prefix_space
      def _SCREAMING_SNAKE_CASE (						self     :  str						,	*lowercase_     :  List[Any]						,	**lowercase_     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		kwargs.get('''is_split_into_words'''						,	lowercase_)
          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._batch_encode_plus(*lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	*lowercase_     :  List[str]						,	**lowercase_     :  List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		kwargs.get('''is_split_into_words'''						,	lowercase_)

          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._encode_plus(*lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		self._tokenizer.model.save(lowercase_						,	name=lowercase_)
          return tuple(lowercase_)



      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  "Conversation"):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		[]
          for is_user, text in conversation.iter_texts():
              input_ids.extend(self.encode(lowercase_						,	add_special_tokens=lowercase_) + [self.eos_token_id])

          if len(lowercase_) > self.model_max_length:
              SCREAMING_SNAKE_CASE_      :  Any    						=		input_ids[-self.model_max_length :]
          return input_ids


