



"""simple docstring"""
import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


A:   Optional[Any]								=		logging.get_logger(__name__)

A:   Dict								=		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A:   Union[str, Any]								=		{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

A:   int								=		{
    '''gpt2''': 1_0_2_4,
    '''gpt2-medium''': 1_0_2_4,
    '''gpt2-large''': 1_0_2_4,
    '''gpt2-xl''': 1_0_2_4,
    '''distilgpt2''': 1_0_2_4,
}



class       SCREAMING_SNAKE_CASE__  (					lowerCAmelCase__ ):
   __lowerCAmelCase  :   Optional[int]						=						VOCAB_FILES_NAMES
   __lowerCAmelCase  :   Optional[Any]						=						PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase  :   Optional[Any]						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase  :   Tuple						=						["""input_ids""", """attention_mask"""]
   __lowerCAmelCase  :   str						=						GPTaTokenizer

   def __init__(   self      ,     _SCREAMING_SNAKE_CASE=None      ,     _SCREAMING_SNAKE_CASE=None      ,     _SCREAMING_SNAKE_CASE=None      ,     _SCREAMING_SNAKE_CASE="<|endoftext|>"      ,     _SCREAMING_SNAKE_CASE="<|endoftext|>"      ,     _SCREAMING_SNAKE_CASE="<|endoftext|>"      ,     _SCREAMING_SNAKE_CASE=False      ,     **_SCREAMING_SNAKE_CASE      ,     )  ->      List[str]:





    '''simple docstring'''




    super().__init__(
        lowercase_      ,     lowercase_      ,     tokenizer_file=lowercase_      ,     unk_token=lowercase_      ,     bos_token=lowercase_      ,     eos_token=lowercase_      ,     add_prefix_space=lowercase_      ,     **lowercase_      ,     )

    UpperCAmelCase	:					List[str]	=	kwargs.pop("""add_bos_token"""      ,     lowercase_						)

    UpperCAmelCase	:					Optional[int]	=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
    if pre_tok_state.get("""add_prefix_space"""      ,     lowercase_						) != add_prefix_space:
     UpperCAmelCase	:					Tuple	=	getattr(lowercase_      ,     pre_tok_state.pop("""type"""						)						)
     UpperCAmelCase	:					Optional[int]	=	add_prefix_space
     UpperCAmelCase	:					int	=	pre_tok_class(**lowercase_						)

    UpperCAmelCase	:					str	=	add_prefix_space

   def 	SCREAMING_SNAKE_CASE     (   self      ,     *_SCREAMING_SNAKE_CASE      ,     **_SCREAMING_SNAKE_CASE						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					str	=	kwargs.get("""is_split_into_words"""      ,     lowercase_						)
    assert self.add_prefix_space or not is_split_into_words, (
        F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
        "to use it with pretokenized inputs."
    )

    return super()._batch_encode_plus(*lowercase_      ,     **lowercase_						)

   def 	SCREAMING_SNAKE_CASE     (   self      ,     *_SCREAMING_SNAKE_CASE      ,     **_SCREAMING_SNAKE_CASE						)  ->      List[str]:





    '''simple docstring'''




    UpperCAmelCase	:					List[Any]	=	kwargs.get("""is_split_into_words"""      ,     lowercase_						)

    assert self.add_prefix_space or not is_split_into_words, (
        F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
        "to use it with pretokenized inputs."
    )

    return super()._encode_plus(*lowercase_      ,     **lowercase_						)

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE      ,     _SCREAMING_SNAKE_CASE = None						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					Union[str, Any]	=	self._tokenizer.model.save(lowercase_      ,     name=lowercase_						)
    return tuple(lowercase_						)




   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					str	=	[]
    for is_user, text in conversation.iter_texts():
     input_ids.extend(self.encode(lowercase_      ,     add_special_tokens=lowercase_						) + [self.eos_token_id]						)

    if len(lowercase_						) > self.model_max_length:
     UpperCAmelCase	:					Tuple	=	input_ids[-self.model_max_length :]
    return input_ids





"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase__    :  List[Any]							    =	{
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Any							    =	['''DistilBertTokenizerFast''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  int							    =	[
         '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''DistilBertForMaskedLM''',
         '''DistilBertForMultipleChoice''',
         '''DistilBertForQuestionAnswering''',
         '''DistilBertForSequenceClassification''',
         '''DistilBertForTokenClassification''',
         '''DistilBertModel''',
         '''DistilBertPreTrainedModel''',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Dict							    =	[
         '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''TFDistilBertForMaskedLM''',
         '''TFDistilBertForMultipleChoice''',
         '''TFDistilBertForQuestionAnswering''',
         '''TFDistilBertForSequenceClassification''',
         '''TFDistilBertForTokenClassification''',
         '''TFDistilBertMainLayer''',
         '''TFDistilBertModel''',
         '''TFDistilBertPreTrainedModel''',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Tuple							    =	[
         '''FlaxDistilBertForMaskedLM''',
         '''FlaxDistilBertForMultipleChoice''',
         '''FlaxDistilBertForQuestionAnswering''',
         '''FlaxDistilBertForSequenceClassification''',
         '''FlaxDistilBertForTokenClassification''',
         '''FlaxDistilBertModel''',
         '''FlaxDistilBertPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_distilbert import (
         DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         DistilBertConfig,
         DistilBertOnnxConfig,
     )
     from .tokenization_distilbert import DistilBertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_distilbert_fast import DistilBertTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_distilbert import (
              DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
              DistilBertModel,
              DistilBertPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_distilbert import (
              TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFDistilBertForMaskedLM,
              TFDistilBertForMultipleChoice,
              TFDistilBertForQuestionAnswering,
              TFDistilBertForSequenceClassification,
              TFDistilBertForTokenClassification,
              TFDistilBertMainLayer,
              TFDistilBertModel,
              TFDistilBertPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_distilbert import (
              FlaxDistilBertForMaskedLM,
              FlaxDistilBertForMultipleChoice,
              FlaxDistilBertForQuestionAnswering,
              FlaxDistilBertForSequenceClassification,
              FlaxDistilBertForTokenClassification,
              FlaxDistilBertModel,
              FlaxDistilBertPreTrainedModel,
          )

else:
     import sys

     lowercase__    :  Union[str, Any]							    =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





