



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


A__  :  str								=					logging.get_logger(__name__)

A__  :  Dict								=					{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__  :  Tuple								=					{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

A__  :  Optional[int]								=					{
    '''gpt2''': 1024,
    '''gpt2-medium''': 1024,
    '''gpt2-large''': 1024,
    '''gpt2-xl''': 1024,
    '''distilgpt2''': 1024,
}



class 				__snake_case						(     UpperCamelCase_						):
							_a			      = VOCAB_FILES_NAMES
							_a			      = PRETRAINED_VOCAB_FILES_MAP
							_a			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_a			      = ['''input_ids''', '''attention_mask''']
							_a			      = GPTaTokenizer




							def __init__(       self     :       Tuple				,				A_     :       Any=None				,				A_     :       Any=None				,				A_     :       Any=None				,				A_     :       List[Any]="<|endoftext|>"				,				A_     :       Optional[Any]="<|endoftext|>"				,				A_     :       Dict="<|endoftext|>"				,				A_     :       Optional[Any]=False				,				**A_     :       int				,				):
									super().__init__(
									    A_				,				A_				,				tokenizer_file=A_				,				unk_token=A_				,				bos_token=A_				,				eos_token=A_				,				add_prefix_space=A_				,				**A_				,				)

									lowerCAmelCase_			:     str									= kwargs.pop('''add_bos_token'''				,				A_)

									lowerCAmelCase_			:     Optional[int]									= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
									if pre_tok_state.get('''add_prefix_space'''				,				A_) != add_prefix_space:
											lowerCAmelCase_			:     Any									= getattr(A_				,				pre_tok_state.pop('''type'''))
											lowerCAmelCase_			:     List[str]									= add_prefix_space
											lowerCAmelCase_			:     int									= pre_tok_class(**A_)

									lowerCAmelCase_			:     Union[str, Any]									= add_prefix_space




							def    UpperCAmelCase__			(       self     :       Optional[Any]				,				*A_     :       Optional[Any]				,				**A_     :       Union[str, Any]):
									lowerCAmelCase_			:     List[str]									= kwargs.get('''is_split_into_words'''				,				A_)
									assert self.add_prefix_space or not is_split_into_words, (
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    "to use it with pretokenized inputs."
									)

									return super()._batch_encode_plus(*A_				,				**A_)




							def    UpperCAmelCase__			(       self     :       Dict				,				*A_     :       List[str]				,				**A_     :       Union[str, Any]):
									lowerCAmelCase_			:     Union[str, Any]									= kwargs.get('''is_split_into_words'''				,				A_)

									assert self.add_prefix_space or not is_split_into_words, (
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    "to use it with pretokenized inputs."
									)

									return super()._encode_plus(*A_				,				**A_)




							def    UpperCAmelCase__			(       self     :       Optional[int]				,				A_     :       str				,				A_     :       Optional[str] = None):
									lowerCAmelCase_			:     List[Any]									= self._tokenizer.model.save(A_				,				name=A_)
									return tuple(A_)







							def    UpperCAmelCase__			(       self     :       Optional[Any]				,				A_     :       "Conversation"):
									lowerCAmelCase_			:     Optional[Any]									= []
									for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(A_				,				add_special_tokens=A_) + [self.eos_token_id])

									if len(A_) > self.model_max_length:
											lowerCAmelCase_			:     Optional[int]									= input_ids[-self.model_max_length :]
									return input_ids









"""simple docstring"""






from importlib import import_module

from .logging import get_logger


UpperCAmelCase    :     Any   						=		get_logger(__name__)





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Optional[int] ,       UpperCamelCase    : List[Any] ,       UpperCamelCase    : int=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					attrs or []
												if module is not None:
																			for key in module.__dict__:
																										if key in attrs or not key.startswith("""__"""  ):
																																	setattr(self ,       UpperCamelCase ,       getattr(UpperCamelCase ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						Any      							=					module._original_module if isinstance(UpperCamelCase ,       _PatchedModuleObj  ) else module






class      lowerCamelCase__   :







					"""simple docstring"""


					__a	       =							[]






					def __init__(       self    : str ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : str ,       UpperCamelCase    : List[str] ,       UpperCamelCase    : Optional[Any]=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					obj
												__UpperCAmelCase   :						Union[str, Any]      							=					target
												__UpperCAmelCase   :						List[str]      							=					new
												__UpperCAmelCase   :						Optional[int]      							=					target.split("""."""  )[0]
												__UpperCAmelCase   :						Tuple      							=					{}
												__UpperCAmelCase   :						Union[str, Any]      							=					attrs or []






					def __enter__(       self    : Dict  ):
												'''simple docstring'''






												*__UpperCAmelCase							,__UpperCAmelCase   :						str      							=					self.target.split("""."""  )

												# Patch modules:
												# it's used to patch attributes of submodules like "os.path.join";
												# in this case we need to patch "os" and "os.path"

												for i in range(len(UpperCamelCase  )  ):
																			try:
																										__UpperCAmelCase   :						List[Any]      							=					import_module(""".""".join(submodules[: i + 1]  )  )
																			except ModuleNotFoundError:
																										continue
																			# We iterate over all the globals in self.obj in case we find "os" or "os.path"
																			for attr in self.obj.__dir__():
																										__UpperCAmelCase   :						List[Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																										# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
																										# This allows to patch renamed modules like "from os import path as ospath".
																										if obj_attr is submodule or (
																										    (isinstance(UpperCamelCase ,       _PatchedModuleObj  ) and obj_attr._original_module is submodule)
																										):
																																	__UpperCAmelCase   :						Tuple      							=					obj_attr
																																	# patch at top level
																																	setattr(self.obj ,       UpperCamelCase ,       _PatchedModuleObj(UpperCamelCase ,       attrs=self.attrs  )  )
																																	__UpperCAmelCase   :						int      							=					getattr(self.obj ,       UpperCamelCase  )
																																	# construct lower levels patches
																																	for key in submodules[i + 1 :]:
																																								setattr(UpperCamelCase ,       UpperCamelCase ,       _PatchedModuleObj(getattr(UpperCamelCase ,       UpperCamelCase ,       UpperCamelCase  ) ,       attrs=self.attrs  )  )
																																								__UpperCAmelCase   :						Optional[int]      							=					getattr(UpperCamelCase ,       UpperCamelCase  )
																																	# finally set the target attribute
																																	setattr(UpperCamelCase ,       UpperCamelCase ,       self.new  )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

												if submodules:  # if it's an attribute of a submodule like "os.path.join"
																			try:
																										__UpperCAmelCase   :						int      							=					getattr(import_module(""".""".join(UpperCamelCase  )  ) ,       UpperCamelCase  )
																			except (AttributeError, ModuleNotFoundError):
																										return
																			# We iterate over all the globals in self.obj in case we find "os.path.join"
																			for attr in self.obj.__dir__():
																										# We don't check for the name of the global, but rather if its value *is* "os.path.join".
																										# This allows to patch renamed attributes like "from os.path import join as pjoin".
																										if getattr(self.obj ,       UpperCamelCase  ) is attr_value:
																																	__UpperCAmelCase   :						Union[str, Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																																	setattr(self.obj ,       UpperCamelCase ,       self.new  )
												elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
																			__UpperCAmelCase   :						str      							=					globals()["""__builtins__"""][target_attr]
																			setattr(self.obj ,       UpperCamelCase ,       self.new  )
												else:
																			raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''  )






					def __exit__(       self    : str ,       *UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												for attr in list(self.original  ):
																			setattr(self.obj ,       UpperCamelCase ,       self.original.pop(UpperCamelCase  )  )






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												self.__enter__()
												self._active_patches.append(self  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												try:
																			self._active_patches.remove(self  )
												except ValueError:
																			# If the patch hasn't been started this will fail
																			return None

												return self.__exit__()
