


'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_  :				Any						= logging.get_logger(__name__)
a_  :				Union[str, Any]						= {"""vocab_file""": """spiece.model"""}

a_  :				str						= {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

a_  :				Union[str, Any]						= {
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

a_  :				int						= """▁"""





class      snake_case    (    lowercase			):





  """simple docstring"""




  _lowerCamelCase		    =  VOCAB_FILES_NAMES
  _lowerCamelCase		    =  PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase		    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






  def __init__(    self			,  UpperCamelCase			,  UpperCamelCase=True			,  UpperCamelCase=True			,  UpperCamelCase=False			,  UpperCamelCase="[CLS]"			,  UpperCamelCase="[SEP]"			,  UpperCamelCase="<unk>"			,  UpperCamelCase="[SEP]"			,  UpperCamelCase="<pad>"			,  UpperCamelCase="[CLS]"			,  UpperCamelCase="[MASK]"			,  UpperCamelCase = None			,  **UpperCamelCase			,  ):







   """simple docstring"""
   # Mask token behave like a normal word, i.e. include the space before it and
   # is included in the raw text, there should be a match in a non-normalized sentence.
   lowerCamelCase_      =  (
       AddedToken(UpperCamelCase			,  lstrip=UpperCamelCase			,  rstrip=UpperCamelCase			,  normalized=UpperCamelCase     )
       if isinstance(UpperCamelCase			,  UpperCamelCase     )
       else mask_token
   )

   lowerCamelCase_      =  {} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=UpperCamelCase			,  remove_space=UpperCamelCase			,  keep_accents=UpperCamelCase			,  bos_token=UpperCamelCase			,  eos_token=UpperCamelCase			,  unk_token=UpperCamelCase			,  sep_token=UpperCamelCase			,  pad_token=UpperCamelCase			,  cls_token=UpperCamelCase			,  mask_token=UpperCamelCase			,  sp_model_kwargs=self.sp_model_kwargs			,  **UpperCamelCase			,  )

   lowerCamelCase_      =  do_lower_case
   lowerCamelCase_      =  remove_space
   lowerCamelCase_      =  keep_accents
   lowerCamelCase_      =  vocab_file

   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(UpperCamelCase     )






  @property
  def   snake_case		(    self     ):







   """simple docstring"""
   return len(self.sp_model     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  {self.convert_ids_to_tokens(UpperCamelCase     ): i for i in range(self.vocab_size     )}
   vocab.update(self.added_tokens_encoder     )
   return vocab






  def __getstate__(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  self.__dict__.copy()
   lowerCamelCase_      =  None
   return state






  def __setstate__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  d

   # for backward compatibility
   if not hasattr(self			,  "sp_model_kwargs"     ):
    lowerCamelCase_      =  {}

   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(self.vocab_file     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if self.remove_space:
    lowerCamelCase_      =  " ".join(inputs.strip().split()     )
   else:
    lowerCamelCase_      =  inputs
   lowerCamelCase_      =  outputs.replace("``"			,  "\""     ).replace("''"			,  "\""     )

   if not self.keep_accents:
    lowerCamelCase_      =  unicodedata.normalize("NFKD"			,  UpperCamelCase     )
    lowerCamelCase_      =  "".join([c for c in outputs if not unicodedata.combining(UpperCamelCase     )]     )
   if self.do_lower_case:
    lowerCamelCase_      =  outputs.lower()

   return outputs






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.preprocess_text(UpperCamelCase     )
   lowerCamelCase_      =  self.sp_model.encode(UpperCamelCase			,  out_type=UpperCamelCase     )
   lowerCamelCase_      =  []
   for piece in pieces:
    if len(UpperCamelCase     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
     lowerCamelCase_      =  self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase			,  ""     )     )
     if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
      if len(cur_pieces[0]     ) == 1:
       lowerCamelCase_      =  cur_pieces[1:]
      else:
       lowerCamelCase_      =  cur_pieces[0][1:]
     cur_pieces.append(piece[-1]     )
     new_pieces.extend(UpperCamelCase     )
    else:
     new_pieces.append(UpperCamelCase     )

   return new_pieces






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.sp_model.PieceToId(UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.sp_model.IdToPiece(UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  []
   lowerCamelCase_      =  ""
   lowerCamelCase_      =  False
   for token in tokens:
    # make sure that special tokens are not decoded using sentencepiece model
    if token in self.all_special_tokens:
     if not prev_is_special:
      out_string += " "
     out_string += self.sp_model.decode(UpperCamelCase     ) + token
     lowerCamelCase_      =  True
     lowerCamelCase_      =  []
    else:
     current_sub_tokens.append(UpperCamelCase     )
     lowerCamelCase_      =  False
   out_string += self.sp_model.decode(UpperCamelCase     )
   return out_string.strip()






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  [self.sep_token_id]
   lowerCamelCase_      =  [self.cls_token_id]
   if token_ids_a is None:
    return cls + token_ids_a + sep
   return cls + token_ids_a + sep + token_ids_a + sep






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None			,  UpperCamelCase = False     ):







   """simple docstring"""

   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=UpperCamelCase			,  token_ids_a=UpperCamelCase			,  already_has_special_tokens=UpperCamelCase     )

   if token_ids_a is not None:
    return [1] + ([0] * len(UpperCamelCase     )) + [1] + ([0] * len(UpperCamelCase     )) + [1]
   return [1] + ([0] * len(UpperCamelCase     )) + [1]






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  [self.sep_token_id]
   lowerCamelCase_      =  [self.cls_token_id]

   if token_ids_a is None:
    return len(cls + token_ids_a + sep     ) * [0]
   return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   if not os.path.isdir(UpperCamelCase     ):
    logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
    return
   lowerCamelCase_      =  os.path.join(
       UpperCamelCase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

   if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase     ) and os.path.isfile(self.vocab_file     ):
    copyfile(self.vocab_file			,  UpperCamelCase     )
   elif not os.path.isfile(self.vocab_file     ):
    with open(UpperCamelCase			,  "wb"     ) as fi:
     lowerCamelCase_      =  self.sp_model.serialized_model_proto()
     fi.write(UpperCamelCase     )

   return (out_vocab_file,)




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase  							=						logging.get_logger(__name__)

UpperCamelCase  							=						{
    '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/config.json''',
    '''YituTech/conv-bert-medium-small''': (
        '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/config.json'''
    ),
    '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/config.json''',
    # See all ConvBERT models at https://huggingface.co/models?filter=convbert
}



class   snake_case_       (		__A       ):
   __A	:		List[str]         =				"convbert"






   def __init__(      self				:     Union[str, Any]	,						lowercase_				:     str=3_05_22	,						lowercase_				:     Any=7_68	,						lowercase_				:     Tuple=12	,						lowercase_				:     List[str]=12	,						lowercase_				:     Optional[int]=30_72	,						lowercase_				:     Union[str, Any]="gelu"	,						lowercase_				:     str=0.1	,						lowercase_				:     List[str]=0.1	,						lowercase_				:     Optional[Any]=5_12	,						lowercase_				:     Dict=2	,						lowercase_				:     Union[str, Any]=0.02	,						lowercase_				:     Optional[Any]=1E-12	,						lowercase_				:     Optional[int]=1	,						lowercase_				:     List[Any]=0	,						lowercase_				:     Optional[int]=2	,						lowercase_				:     str=7_68	,						lowercase_				:     Dict=2	,						lowercase_				:     Optional[Any]=9	,						lowercase_				:     Union[str, Any]=1	,						lowercase_				:     Any=None	,						**lowercase_				:     Optional[Any]	,						)					->						Dict:
       super().__init__(
           pad_token_id=lowercase_	,						bos_token_id=lowercase_	,						eos_token_id=lowercase_	,						**lowercase_	,						)

       lowercase__ :   List[str]    					=				vocab_size
       lowercase__ :   Union[str, Any]    					=				hidden_size
       lowercase__ :   Any    					=				num_hidden_layers
       lowercase__ :   List[str]    					=				num_attention_heads
       lowercase__ :   Union[str, Any]    					=				intermediate_size
       lowercase__ :   Optional[Any]    					=				hidden_act
       lowercase__ :   int    					=				hidden_dropout_prob
       lowercase__ :   str    					=				attention_probs_dropout_prob
       lowercase__ :   Union[str, Any]    					=				max_position_embeddings
       lowercase__ :   Optional[int]    					=				type_vocab_size
       lowercase__ :   Tuple    					=				initializer_range
       lowercase__ :   List[str]    					=				layer_norm_eps
       lowercase__ :   List[Any]    					=				embedding_size
       lowercase__ :   Optional[Any]    					=				head_ratio
       lowercase__ :   Dict    					=				conv_kernel_size
       lowercase__ :   Tuple    					=				num_groups
       lowercase__ :   Optional[int]    					=				classifier_dropout



class   snake_case_       (		__A       ):






   @property
   def       __UpperCamelCase     (      self				:     Optional[int]  )					->						Mapping[str, Mapping[int, str]]:
       if self.task == "multiple-choice":
           lowercase__ :   Tuple    					=				{0: "batch", 1: "choice", 2: "sequence"}
       else:
           lowercase__ :   str    					=				{0: "batch", 1: "sequence"}
       return OrderedDict(
           [
               ("input_ids", dynamic_axis),
               ("attention_mask", dynamic_axis),
               ("token_type_ids", dynamic_axis),
           ]  )



