






"""simple docstring"""






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_albert import AlbertTokenizer
else:
  UpperCAmelCase_   : Union[str, Any]   			= None

UpperCAmelCase_   : List[str]   			= logging.get_logger(__name__)
UpperCAmelCase_   : Union[str, Any]   			= {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_   : Optional[Any]   			= {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json""",
    },
}

UpperCAmelCase_   : Any   			= {
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

UpperCAmelCase_   : Union[str, Any]   			= """▁"""




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  AlbertTokenizer
      def __init__(						self     :  List[Any]						,	lowercase_     :  List[Any]=None						,	lowercase_     :  Optional[Any]=None						,	lowercase_     :  Dict=True						,	lowercase_     :  Optional[Any]=True						,	lowercase_     :  Optional[int]=False						,	lowercase_     :  List[str]="[CLS]"						,	lowercase_     :  str="[SEP]"						,	lowercase_     :  Dict="<unk>"						,	lowercase_     :  int="[SEP]"						,	lowercase_     :  List[str]="<pad>"						,	lowercase_     :  Optional[Any]="[CLS]"						,	lowercase_     :  Optional[int]="[MASK]"						,	**lowercase_     :  List[str]						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		(
              AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_						,	normalized=lowercase_)
              if isinstance(lowercase_						,	lowercase_)
              else mask_token
          )

          super().__init__(
              lowercase_						,	tokenizer_file=lowercase_						,	do_lower_case=lowercase_						,	remove_space=lowercase_						,	keep_accents=lowercase_						,	bos_token=lowercase_						,	eos_token=lowercase_						,	unk_token=lowercase_						,	sep_token=lowercase_						,	pad_token=lowercase_						,	cls_token=lowercase_						,	mask_token=lowercase_						,	**lowercase_						,	)

          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		do_lower_case
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		remove_space
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		keep_accents
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		vocab_file
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		False if not self.vocab_file else True
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		[self.sep_token_id]
          SCREAMING_SNAKE_CASE_      :  str    						=		[self.cls_token_id]
          if token_ids_a is None:
              return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep
      def _SCREAMING_SNAKE_CASE (						self     :  Union[str, Any]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		[self.sep_token_id]
          SCREAMING_SNAKE_CASE_      :  Dict    						=		[self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



      def _SCREAMING_SNAKE_CASE (						self     :  List[str]						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          if not self.can_save_slow_tokenizer:
              raise ValueError(
                  '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                  '''tokenizer.''')

          if not os.path.isdir(lowercase_):
              logger.error(F'Vocabulary path ({save_directory}) should be a directory')
              return
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		os.path.join(
              lowercase_						,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

          if os.path.abspath(self.vocab_file) != os.path.abspath(lowercase_):
              copyfile(self.vocab_file						,	lowercase_)

          return (out_vocab_file,)










"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_   : Optional[int]   			= logging.get_logger(__name__)

UpperCAmelCase_   : Any   			= {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/config.json"""}




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  "openai-gpt"
      __UpperCamelCase	 =  {
          "max_position_embeddings": "n_positions",
          "hidden_size": "n_embd",
          "num_attention_heads": "n_head",
          "num_hidden_layers": "n_layer",
      }
      def __init__(						self     :  List[str]						,	lowercase_     :  List[str]=40478						,	lowercase_     :  List[str]=512						,	lowercase_     :  Optional[Any]=768						,	lowercase_     :  Tuple=12						,	lowercase_     :  Tuple=12						,	lowercase_     :  Union[str, Any]="gelu"						,	lowercase_     :  List[Any]=0.1						,	lowercase_     :  Tuple=0.1						,	lowercase_     :  List[Any]=0.1						,	lowercase_     :  List[Any]=1e-5						,	lowercase_     :  int=0.02						,	lowercase_     :  Optional[int]="cls_index"						,	lowercase_     :  Any=True						,	lowercase_     :  List[Any]=None						,	lowercase_     :  List[str]=True						,	lowercase_     :  Optional[Any]=0.1						,	**lowercase_     :  List[str]						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		vocab_size
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		n_positions
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		n_embd
          SCREAMING_SNAKE_CASE_      :  Dict    						=		n_layer
          SCREAMING_SNAKE_CASE_      :  Any    						=		n_head
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		afn
          SCREAMING_SNAKE_CASE_      :  int    						=		resid_pdrop
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		embd_pdrop
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		attn_pdrop
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		layer_norm_epsilon
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		initializer_range
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		summary_type
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		summary_use_proj
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		summary_activation
          SCREAMING_SNAKE_CASE_      :  Any    						=		summary_first_dropout
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		summary_proj_to_labels
          super().__init__(**lowercase_)


