


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_snake_case             =     logging.get_logger(__name__)

_snake_case             =     {'''vocab_file''': '''spiece.model'''}

_snake_case             =     {
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}
class 	UpperCAmelCase_					(					UpperCamelCase    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A=False						,				__A=True						,				__A=False						,				__A="<s>"						,				__A="</s>"						,				__A="<unk>"						,				__A="<sep>"						,				__A="<pad>"						,				__A="<cls>"						,				__A="<mask>"						,				__A=["<eop>", "<eod>"]						,				__A = None						,				**__A						,				):






     """simple docstring"""





     lowerCamelCase :					List[str]    =				AddedToken(__A						,				lstrip=__A						,				rstrip=__A ) if isinstance(__A						,				__A ) else mask_token

     lowerCamelCase :					Tuple    =				{} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         do_lower_case=__A						,				remove_space=__A						,				keep_accents=__A						,				bos_token=__A						,				eos_token=__A						,				unk_token=__A						,				sep_token=__A						,				pad_token=__A						,				cls_token=__A						,				mask_token=__A						,				additional_special_tokens=__A						,				sp_model_kwargs=self.sp_model_kwargs						,				**__A						,				)

     lowerCamelCase :					Dict    =				3

     lowerCamelCase :					Dict    =				do_lower_case
     lowerCamelCase :					Optional[int]    =				remove_space
     lowerCamelCase :					Any    =				keep_accents
     lowerCamelCase :					List[Any]    =				vocab_file

     lowerCamelCase :					str    =				spm.SentencePieceProcessor(**self.sp_model_kwargs )
     self.sp_model.Load(__A )

     try:
       import jieba
     except ModuleNotFoundError as error:
       raise error.__class__(
           "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
           "See https://pypi.org/project/jieba/ for installation." )
     lowerCamelCase :					Any    =				jieba
     lowerCamelCase :					int    =				str.maketrans(" \n"						,				"\u2582\u2583" )






   @property
   # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
   def _snake_case    (		self ):






     """simple docstring"""





     return len(self.sp_model )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					int    =				{self.convert_ids_to_tokens(__A ): i for i in range(self.vocab_size )}
     vocab.update(self.added_tokens_encoder )
     return vocab






   def __getstate__(		self ):






     """simple docstring"""





     lowerCamelCase :					str    =				self.__dict__.copy()
     lowerCamelCase :					Union[str, Any]    =				None
     return state






   def __setstate__(		self						,				__A ):






     """simple docstring"""





     lowerCamelCase :					int    =				d

     # for backward compatibility
     if not hasattr(self						,				"sp_model_kwargs" ):
       lowerCamelCase :					int    =				{}

     lowerCamelCase :					Dict    =				spm.SentencePieceProcessor(**self.sp_model_kwargs )
     self.sp_model.Load(self.vocab_file )






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     if self.remove_space:
       lowerCamelCase :					Tuple    =				" ".join(inputs.strip().split() )
     else:
       lowerCamelCase :					List[Any]    =				inputs
     lowerCamelCase :					List[str]    =				outputs.replace("``"						,				"\"" ).replace("''"						,				"\"" )

     if not self.keep_accents:
       lowerCamelCase :					int    =				unicodedata.normalize("NFKD"						,				__A )
       lowerCamelCase :					Dict    =				"".join([c for c in outputs if not unicodedata.combining(__A )] )
     if self.do_lower_case:
       lowerCamelCase :					List[Any]    =				outputs.lower()

     return outputs






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     lowerCamelCase :					Optional[int]    =				self.preprocess_text(__A )
     lowerCamelCase :					List[Any]    =				self.sp_model.encode(__A						,				out_type=__A )
     lowerCamelCase :					int    =				[]
     for piece in pieces:
       if len(__A ) > 1 and piece[-1] == str("," ) and piece[-2].isdigit():
         lowerCamelCase :					Dict    =				self.sp_model.EncodeAsPieces(piece[:-1].replace(__A						,				"" ) )
         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
           if len(cur_pieces[0] ) == 1:
             lowerCamelCase :					List[str]    =				cur_pieces[1:]
           else:
             lowerCamelCase :					List[Any]    =				cur_pieces[0][1:]
         cur_pieces.append(piece[-1] )
         new_pieces.extend(__A )
       else:
         new_pieces.append(__A )

     return new_pieces






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     return self.sp_model.PieceToId(__A )






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     return self.sp_model.IdToPiece(__A )






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     lowerCamelCase :					Any    =				"".join(__A ).replace(__A						,				" " ).strip()
     return out_string






   def _snake_case    (		self						,				__A						,				__A = None ):






     """simple docstring"""





     lowerCamelCase :					List[str]    =				[self.sep_token_id]
     lowerCamelCase :					List[str]    =				[self.cls_token_id]
     if token_ids_a is None:
       return token_ids_a + sep + cls
     return token_ids_a + sep + token_ids_a + sep + cls






   def _snake_case    (		self						,				__A						,				__A = None						,				__A = False ):






     """simple docstring"""





     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__A						,				token_ids_a=__A						,				already_has_special_tokens=__A )

     if token_ids_a is not None:
       return ([0] * len(__A )) + [1] + ([0] * len(__A )) + [1, 1]
     return ([0] * len(__A )) + [1, 1]






   def _snake_case    (		self						,				__A						,				__A = None ):






     """simple docstring"""





     lowerCamelCase :					Dict    =				[self.sep_token_id]
     lowerCamelCase :					Optional[int]    =				[2]

     if token_ids_a is None:
       return len(token_ids_a + sep ) * [0] + cls_segment_id
     return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id






   def _snake_case    (		self						,				__A						,				__A = None ):






     """simple docstring"""





     if not os.path.isdir(__A ):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
       return
     lowerCamelCase :					List[str]    =				os.path.join(
         __A						,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

     if os.path.abspath(self.vocab_file ) != os.path.abspath(__A ) and os.path.isfile(self.vocab_file ):
       copyfile(self.vocab_file						,				__A )
     elif not os.path.isfile(self.vocab_file ):
       with open(__A						,				"wb" ) as fi:
         lowerCamelCase :					Optional[Any]    =				self.sp_model.serialized_model_proto()
         fi.write(__A )

     return (out_vocab_file,)






   def _snake_case    (		self						,				*__A						,				**__A ):






     """simple docstring"""





     lowerCamelCase :					Union[str, Any]    =				super()._decode(*__A						,				**__A )
     lowerCamelCase :					int    =				text.replace(" "						,				"" ).replace("\u2582"						,				" " ).replace("\u2583"						,				"\n" )
     return text







from ..utils import DummyObject, requires_backends
class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Any     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[int]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      str     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      List[Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[int]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Dict     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Dict     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Union[str, Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[int]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Any     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      int     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



