





'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a   	=    logging.get_logger(__name__)
a   	=    {"vocab_file": "spiece.model"}

a   	=    {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

a   	=    {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

a   	=    "▁"







class __a			( _snake_case      ):
   __UpperCamelCase :			List[Any]						   =						VOCAB_FILES_NAMES
   __UpperCamelCase :			Optional[Any]						   =						PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase :			Dict						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





   def __init__(       self					:      Optional[int]					,lowerCamelCase					:      str					,lowerCamelCase					:      Optional[Any]=True					,lowerCamelCase					:      Tuple=True					,lowerCamelCase					:      List[str]=False					,lowerCamelCase					:      Dict="[CLS]"					,lowerCamelCase					:      List[str]="[SEP]"					,lowerCamelCase					:      List[str]="<unk>"					,lowerCamelCase					:      Tuple="[SEP]"					,lowerCamelCase					:      Tuple="<pad>"					,lowerCamelCase					:      str="[CLS]"					,lowerCamelCase					:      Optional[Any]="[MASK]"					,lowerCamelCase					:      Optional[Dict[str, Any]] = None					,**lowerCamelCase					:      Dict					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							(
          AddedToken(lowerCamelCase					,lstrip=lowerCamelCase					,rstrip=lowerCamelCase					,normalized=lowerCamelCase     )
          if isinstance(lowerCamelCase					,lowerCamelCase     )
          else mask_token
      )

      __SCREAMING_SNAKE_CASE   				=							{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=lowerCamelCase					,remove_space=lowerCamelCase					,keep_accents=lowerCamelCase					,bos_token=lowerCamelCase					,eos_token=lowerCamelCase					,unk_token=lowerCamelCase					,sep_token=lowerCamelCase					,pad_token=lowerCamelCase					,cls_token=lowerCamelCase					,mask_token=lowerCamelCase					,sp_model_kwargs=self.sp_model_kwargs					,**lowerCamelCase					,)

      __SCREAMING_SNAKE_CASE   				=							do_lower_case
      __SCREAMING_SNAKE_CASE   				=							remove_space
      __SCREAMING_SNAKE_CASE   				=							keep_accents
      __SCREAMING_SNAKE_CASE   				=							vocab_file

      __SCREAMING_SNAKE_CASE   				=							spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(lowerCamelCase     )





   @property
   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      return len(self.sp_model     )





   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							{self.convert_ids_to_tokens(lowerCamelCase     ): i for i in range(self.vocab_size     )}
      vocab.update(self.added_tokens_encoder     )
      return vocab





   def __getstate__(       self					:      Union[str, Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.__dict__.copy()
      __SCREAMING_SNAKE_CASE   				=							None
      return state





   def __setstate__(       self					:      Optional[Any]					,lowerCamelCase					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							d

      # for backward compatibility
      if not hasattr(self					,"""sp_model_kwargs"""     ):
         __SCREAMING_SNAKE_CASE   				=							{}

      __SCREAMING_SNAKE_CASE   				=							spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(self.vocab_file     )





   def   UpperCAmelCase__  (       self					:      List[Any]					,lowerCamelCase					:      int     ):

      '''simple docstring'''

      if self.remove_space:
         __SCREAMING_SNAKE_CASE   				=							""" """.join(inputs.strip().split()     )
      else:
         __SCREAMING_SNAKE_CASE   				=							inputs
      __SCREAMING_SNAKE_CASE   				=							outputs.replace("""``"""					,"""\""""     ).replace("""''"""					,"""\""""     )

      if not self.keep_accents:
         __SCREAMING_SNAKE_CASE   				=							unicodedata.normalize("""NFKD"""					,lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							"""""".join([c for c in outputs if not unicodedata.combining(lowerCamelCase     )]     )
      if self.do_lower_case:
         __SCREAMING_SNAKE_CASE   				=							outputs.lower()

      return outputs





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.preprocess_text(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.sp_model.encode(lowerCamelCase					,out_type=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							[]
      for piece in pieces:
         if len(lowerCamelCase     ) > 1 and piece[-1] == str(""","""     ) and piece[-2].isdigit():
            __SCREAMING_SNAKE_CASE   				=							self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase					,""""""     )     )
            if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
               if len(cur_pieces[0]     ) == 1:
                  __SCREAMING_SNAKE_CASE   				=							cur_pieces[1:]
               else:
                  __SCREAMING_SNAKE_CASE   				=							cur_pieces[0][1:]
            cur_pieces.append(piece[-1]     )
            new_pieces.extend(lowerCamelCase     )
         else:
            new_pieces.append(lowerCamelCase     )

      return new_pieces





   def   UpperCAmelCase__  (       self					:      Tuple					,lowerCamelCase					:      Union[str, Any]     ):

      '''simple docstring'''

      return self.sp_model.PieceToId(lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Optional[int]					,lowerCamelCase					:      Union[str, Any]     ):

      '''simple docstring'''

      return self.sp_model.IdToPiece(lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      int					,lowerCamelCase					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							""""""
      __SCREAMING_SNAKE_CASE   				=							False
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            if not prev_is_special:
               out_string += " "
            out_string += self.sp_model.decode(lowerCamelCase     ) + token
            __SCREAMING_SNAKE_CASE   				=							True
            __SCREAMING_SNAKE_CASE   				=							[]
         else:
            current_sub_tokens.append(lowerCamelCase     )
            __SCREAMING_SNAKE_CASE   				=							False
      out_string += self.sp_model.decode(lowerCamelCase     )
      return out_string.strip()





   def   UpperCAmelCase__  (       self					:      Optional[int]					,lowerCamelCase					:      List[int]					,lowerCamelCase					:      Optional[List[int]] = None     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[self.sep_token_id]
      __SCREAMING_SNAKE_CASE   				=							[self.cls_token_id]
      if token_ids_a is None:
         return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep





   def   UpperCAmelCase__  (       self					:      int					,lowerCamelCase					:      List[int]					,lowerCamelCase					:      Optional[List[int]] = None					,lowerCamelCase					:      bool = False     ):

      '''simple docstring'''

      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase					,token_ids_a=lowerCamelCase					,already_has_special_tokens=lowerCamelCase     )

      if token_ids_a is not None:
         return [1] + ([0] * len(lowerCamelCase     )) + [1] + ([0] * len(lowerCamelCase     )) + [1]
      return [1] + ([0] * len(lowerCamelCase     )) + [1]





   def   UpperCAmelCase__  (       self					:      Union[str, Any]					,lowerCamelCase					:      List[int]					,lowerCamelCase					:      Optional[List[int]] = None     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[self.sep_token_id]
      __SCREAMING_SNAKE_CASE   				=							[self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep     ) * [0]
      return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]





   def   UpperCAmelCase__  (       self					:      Union[str, Any]					,lowerCamelCase					:      str					,lowerCamelCase					:      Optional[str] = None     ):

      '''simple docstring'''

      if not os.path.isdir(lowerCamelCase     ):
         logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
         return
      __SCREAMING_SNAKE_CASE   				=							os.path.join(
          lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

      if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowerCamelCase     ) and os.path.isfile(self.vocab_file     ):
         copyfile(self.vocab_file					,lowerCamelCase     )
      elif not os.path.isfile(self.vocab_file     ):
         with open(lowerCamelCase					,"""wb"""     ) as fi:
            __SCREAMING_SNAKE_CASE   				=							self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase     )

      return (out_vocab_file,)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


a  :      Any            =    logging.get_logger(__name__)

a  :      List[str]            =    {
    """transfo-xl-wt103""": """https://huggingface.co/transfo-xl-wt103/resolve/main/config.json""",
}

class 					_lowercase    (     UpperCAmelCase__			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Optional[Any]          =    'transfo-xl'
    SCREAMING_SNAKE_CASE:				List[Any]          =    ['mems']
    SCREAMING_SNAKE_CASE:				List[Any]          =    {
        'n_token': 'vocab_size',
        'hidden_size': 'd_model',
        'num_attention_heads': 'n_head',
        'num_hidden_layers': 'n_layer',
    }




    def __init__(    self       ,						lowerCamelCase__=267_735       ,						lowerCamelCase__=[20_000, 40_000, 200_000]       ,						lowerCamelCase__=1_024       ,						lowerCamelCase__=1_024       ,						lowerCamelCase__=16       ,						lowerCamelCase__=64       ,						lowerCamelCase__=4_096       ,						lowerCamelCase__=4       ,						lowerCamelCase__=False       ,						lowerCamelCase__=18       ,						lowerCamelCase__=1_600       ,						lowerCamelCase__=1_000       ,						lowerCamelCase__=True       ,						lowerCamelCase__=True       ,						lowerCamelCase__=0       ,						lowerCamelCase__=-1       ,						lowerCamelCase__=True       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__=0.0       ,						lowerCamelCase__=True       ,						lowerCamelCase__="normal"       ,						lowerCamelCase__=0.0_1       ,						lowerCamelCase__=0.0_1       ,						lowerCamelCase__=0.0_2       ,						lowerCamelCase__=1E-5       ,						lowerCamelCase__=0       ,						**lowerCamelCase__       ,						):
       lowerCAmelCase_:							Dict       =	vocab_size
       lowerCAmelCase_:							Any       =	[]
       self.cutoffs.extend(lowerCamelCase__ )
       if proj_share_all_but_first:
          lowerCAmelCase_:							int       =	[False] + [True] * len(self.cutoffs )
       else:
          lowerCAmelCase_:							Any       =	[False] + [False] * len(self.cutoffs )
       lowerCAmelCase_:							Dict       =	d_model
       lowerCAmelCase_:							Union[str, Any]       =	d_embed
       lowerCAmelCase_:							Dict       =	d_head
       lowerCAmelCase_:							Dict       =	d_inner
       lowerCAmelCase_:							List[str]       =	div_val
       lowerCAmelCase_:							List[str]       =	pre_lnorm
       lowerCAmelCase_:							List[str]       =	n_layer
       lowerCAmelCase_:							List[str]       =	n_head
       lowerCAmelCase_:							str       =	mem_len
       lowerCAmelCase_:							Any       =	same_length
       lowerCAmelCase_:							Optional[Any]       =	attn_type
       lowerCAmelCase_:							int       =	clamp_len
       lowerCAmelCase_:							Optional[int]       =	sample_softmax
       lowerCAmelCase_:							Optional[Any]       =	adaptive
       lowerCAmelCase_:							Optional[int]       =	dropout
       lowerCAmelCase_:							Union[str, Any]       =	dropatt
       lowerCAmelCase_:							Tuple       =	untie_r
       lowerCAmelCase_:							Union[str, Any]       =	init
       lowerCAmelCase_:							Optional[Any]       =	init_range
       lowerCAmelCase_:							Optional[Any]       =	proj_init_std
       lowerCAmelCase_:							Tuple       =	init_std
       lowerCAmelCase_:							Tuple       =	layer_norm_epsilon
       super().__init__(eos_token_id=lowerCamelCase__       ,						**lowerCamelCase__ )




    @property
    def      _a    (    self ):
       # Message copied from Transformer-XL documentation
       logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.''' )
       return -1




    @max_position_embeddings.setter
    def      _a    (    self       ,						lowerCamelCase__ ):
       # Message copied from Transformer-XL documentation
       raise NotImplementedError(
           F'''The model {self.model_type} is one of the few models that has no sequence length limit.''' )