





"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowerCamelCase_     :						Tuple						=					logging.get_logger(__name__)

lowerCamelCase_     :						Optional[Any]						=					{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase_     :						int						=					{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/tokenizer.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/tokenizer.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/tokenizer.json',
    },
}

lowerCamelCase_     :						Union[str, Any]						=					{
    'gpt2': 10_24,
    'gpt2-medium': 10_24,
    'gpt2-large': 10_24,
    'gpt2-xl': 10_24,
    'distilgpt2': 10_24,
}

class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''
      lowercase_      : Optional[int]	    =   VOCAB_FILES_NAMES
      lowercase_      : List[Any]	    =   PRETRAINED_VOCAB_FILES_MAP
      lowercase_      : List[str]	    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowercase_      : Any	    =   ["""input_ids""", """attention_mask"""]
      lowercase_      : Union[str, Any]	    =   GPTaTokenizer

      def __init__(     self					,      snake_case_=None					,      snake_case_=None					,      snake_case_=None					,      snake_case_="<|endoftext|>"					,      snake_case_="<|endoftext|>"					,      snake_case_="<|endoftext|>"					,      snake_case_=False					,      **snake_case_					,      ):

        """simple docstring"""

        super().__init__(
            snake_case_					,      snake_case_					,      tokenizer_file=snake_case_					,      unk_token=snake_case_					,      bos_token=snake_case_					,      eos_token=snake_case_					,      add_prefix_space=snake_case_					,      **snake_case_					,      )

        A_  :       Union[str, Any]										=  kwargs.pop('add_bos_token'					,      snake_case_  )

        A_  :       Union[str, Any]										=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
        if pre_tok_state.get('add_prefix_space'					,      snake_case_  ) != add_prefix_space:
          A_  :       List[str]										=  getattr(snake_case_					,      pre_tok_state.pop('type'  )  )
          A_  :       Optional[int]										=  add_prefix_space
          A_  :       Dict										=  pre_tok_class(**snake_case_  )

        A_  :       Tuple										=  add_prefix_space

      def 					lowerCamelCase_			(     self					,      *snake_case_					,      **snake_case_  ):

        """simple docstring"""

        A_  :       List[str]										=  kwargs.get('is_split_into_words'					,      snake_case_  )
        assert self.add_prefix_space or not is_split_into_words, (
            F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*snake_case_					,      **snake_case_  )

      def 					lowerCamelCase_			(     self					,      *snake_case_					,      **snake_case_  ):

        """simple docstring"""

        A_  :       Tuple										=  kwargs.get('is_split_into_words'					,      snake_case_  )

        assert self.add_prefix_space or not is_split_into_words, (
            F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*snake_case_					,      **snake_case_  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_ = None  ):

        """simple docstring"""

        A_  :       Any										=  self._tokenizer.model.save(snake_case_					,      name=snake_case_  )
        return tuple(snake_case_  )

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        A_  :       Tuple										=  []
        for is_user, text in conversation.iter_texts():
          input_ids.extend(self.encode(snake_case_					,      add_special_tokens=snake_case_  ) + [self.eos_token_id]  )

        if len(snake_case_  ) > self.model_max_length:
          A_  :       Optional[int]										=  input_ids[-self.model_max_length :]
        return input_ids






"""simple docstring"""




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase_     :						str						=					logging.get_logger(__name__)

lowerCamelCase_     :						Any						=					{
    'facebook/xmod-base': 'https://huggingface.co/facebook/xmod-base/resolve/main/config.json',
    'facebook/xmod-large-prenorm': 'https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json',
    'facebook/xmod-base-13-125k': 'https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json',
    'facebook/xmod-base-30-125k': 'https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json',
    'facebook/xmod-base-30-195k': 'https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json',
    'facebook/xmod-base-60-125k': 'https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json',
    'facebook/xmod-base-60-265k': 'https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json',
    'facebook/xmod-base-75-125k': 'https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json',
    'facebook/xmod-base-75-269k': 'https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json',
}

class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''
      lowercase_      : Tuple	    =   """xmod"""

      def __init__(     self					,      snake_case_=3_0_5_2_2					,      snake_case_=7_6_8					,      snake_case_=1_2					,      snake_case_=1_2					,      snake_case_=3_0_7_2					,      snake_case_="gelu"					,      snake_case_=0.1					,      snake_case_=0.1					,      snake_case_=5_1_2					,      snake_case_=2					,      snake_case_=0.02					,      snake_case_=1E-12					,      snake_case_=1					,      snake_case_=0					,      snake_case_=2					,      snake_case_="absolute"					,      snake_case_=True					,      snake_case_=None					,      snake_case_=False					,      snake_case_=2					,      snake_case_=False					,      snake_case_=True					,      snake_case_=True					,      snake_case_=("en_XX",)					,      snake_case_=None					,      **snake_case_					,      ):

        """simple docstring"""

        super().__init__(pad_token_id=snake_case_					,      bos_token_id=snake_case_					,      eos_token_id=snake_case_					,      **snake_case_  )

        A_  :       Union[str, Any]										=  vocab_size
        A_  :       Any										=  hidden_size
        A_  :       List[str]										=  num_hidden_layers
        A_  :       Tuple										=  num_attention_heads
        A_  :       int										=  hidden_act
        A_  :       Any										=  intermediate_size
        A_  :       Any										=  hidden_dropout_prob
        A_  :       Dict										=  attention_probs_dropout_prob
        A_  :       Union[str, Any]										=  max_position_embeddings
        A_  :       List[Any]										=  type_vocab_size
        A_  :       List[str]										=  initializer_range
        A_  :       Any										=  layer_norm_eps
        A_  :       Optional[Any]										=  position_embedding_type
        A_  :       int										=  use_cache
        A_  :       Dict										=  classifier_dropout
        A_  :       int										=  pre_norm
        A_  :       Optional[Any]										=  adapter_reduction_factor
        A_  :       List[Any]										=  adapter_layer_norm
        A_  :       int										=  adapter_reuse_layer_norm
        A_  :       Dict										=  ln_before_adapter
        A_  :       List[str]										=  list(snake_case_  )
        A_  :       Union[str, Any]										=  default_language


class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''

      @property
      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        if self.task == "multiple-choice":
          A_  :       Dict										=  {0: 'batch', 1: 'choice', 2: 'sequence'}
        else:
          A_  :       int										=  {0: 'batch', 1: 'sequence'}
        return OrderedDict(
            [
                ('input_ids', dynamic_axis),
                ('attention_mask', dynamic_axis),
            ]  )