
'''simple docstring'''
import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__						       =      logging.get_logger(__name__)

lowercase__						       =      "▁"

lowercase__						       =      {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

lowercase__						       =      {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

lowercase__						       =      {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

lowercase__						       =      {
    "ernie-m-base": 5_1_4,
    "ernie-m-large": 5_1_4,
}

lowercase__						       =      {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}






class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""

		lowerCamelCase     							=						["input_ids"]

		lowerCamelCase     							=						VOCAB_FILES_NAMES
		lowerCamelCase     							=						PRETRAINED_INIT_CONFIGURATION
		lowerCamelCase     							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowerCamelCase     							=						PRETRAINED_VOCAB_FILES_MAP
		lowerCamelCase     							=						RESOURCE_FILES_NAMES



		def __init__(      self				:			int  ,	UpperCamelCase__				:			Optional[int]  ,	UpperCamelCase__				:			List[Any]=None  ,	UpperCamelCase__				:			Optional[Any]=False  ,	UpperCamelCase__				:			Dict="utf8"  ,	UpperCamelCase__				:			Optional[Any]="[UNK]"  ,	UpperCamelCase__				:			Optional[Any]="[SEP]"  ,	UpperCamelCase__				:			Union[str, Any]="[PAD]"  ,	UpperCamelCase__				:			Tuple="[CLS]"  ,	UpperCamelCase__				:			Union[str, Any]="[MASK]"  ,	UpperCamelCase__				:			Optional[Dict[str, Any]] = None  ,	**UpperCamelCase__				:			Any  ,	)						->			None:






				"""simple docstring"""







				snake_case  :       Tuple     			=						{} if sp_model_kwargs is None else sp_model_kwargs
				super().__init__(
				    do_lower_case=UpperCamelCase__  ,	unk_token=UpperCamelCase__  ,	sep_token=UpperCamelCase__  ,	pad_token=UpperCamelCase__  ,	cls_token=UpperCamelCase__  ,	mask_token=UpperCamelCase__  ,	vocab_file=UpperCamelCase__  ,	encoding=UpperCamelCase__  ,	sp_model_kwargs=self.sp_model_kwargs  ,	**UpperCamelCase__  ,	)
				snake_case  :       Tuple     			=						do_lower_case
				snake_case  :       Optional[int]     			=						sentencepiece_model_ckpt
				snake_case  :       List[str]     			=						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
				self.sp_model.Load(UpperCamelCase__      )

				# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
				if vocab_file is not None:
						snake_case  :       List[str]     			=						self.load_vocab(filepath=UpperCamelCase__      )
				else:
						snake_case  :       Optional[int]     			=						{self.sp_model.id_to_piece(UpperCamelCase__      ): id for id in range(self.sp_model.get_piece_size()      )}
				snake_case  :       List[str]     			=						{v: k for k, v in self.vocab.items()}



		def       lowerCAmelCase				(      self				:			Optional[Any]  ,	UpperCamelCase__				:			Optional[int]      )						->			int:






				"""simple docstring"""







				if text is None:
						return None

				snake_case  :       List[str]     			=						self.tokenize(UpperCamelCase__      )
				snake_case			,snake_case  :       Optional[int]     			=						'''''', []

				for i, ch in enumerate(UpperCamelCase__      ):
						if ch in self.SP_CHAR_MAPPING:
								snake_case  :       Optional[int]     			=						self.SP_CHAR_MAPPING.get(UpperCamelCase__      )
						else:
								snake_case  :       int     			=						unicodedata.normalize('''NFKC'''  ,	UpperCamelCase__      )
						if self.is_whitespace(UpperCamelCase__      ):
								continue
						normalized_text += ch
						char_mapping.extend([i] * len(UpperCamelCase__      )      )

				snake_case			,snake_case			,snake_case  :       Union[str, Any]     			=						normalized_text, [], 0

				if self.do_lower_case:
						snake_case  :       Optional[int]     			=						text.lower()

				for token in split_tokens:
						if token[:1] == "▁":
								snake_case  :       Any     			=						token[1:]
						snake_case  :       Optional[int]     			=						text[offset:].index(UpperCamelCase__      ) + offset
						snake_case  :       Optional[Any]     			=						start + len(UpperCamelCase__      )

						token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)      )
						snake_case  :       int     			=						end
				return token_mapping



		@property
		def       lowerCAmelCase				(      self				:			int      )						->			List[Any]:






				"""simple docstring"""







				return len(self.vocab      )



		def       lowerCAmelCase				(      self				:			Union[str, Any]      )						->			List[str]:






				"""simple docstring"""







				return dict(self.vocab  ,	**self.added_tokens_encoder      )



		def __getstate__(      self				:			List[Any]      )						->			List[str]:






				"""simple docstring"""







				snake_case  :       Union[str, Any]     			=						self.__dict__.copy()
				snake_case  :       Optional[int]     			=						None
				return state



		def __setstate__(      self				:			str  ,	UpperCamelCase__				:			Optional[Any]      )						->			Optional[Any]:






				"""simple docstring"""







				snake_case  :       List[Any]     			=						d

				# for backward compatibility
				if not hasattr(self  ,	'''sp_model_kwargs'''      ):
						snake_case  :       List[Any]     			=						{}

				snake_case  :       List[Any]     			=						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
				self.sp_model.Load(self.sentencepiece_model_ckpt      )



		def       lowerCAmelCase				(      self				:			int  ,	UpperCamelCase__				:			Tuple      )						->			List[Any]:






				"""simple docstring"""







				return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase__  ,	UpperCamelCase__      ) for c in text)      )



		def       lowerCAmelCase				(      self				:			str  ,	UpperCamelCase__				:			Dict  ,	UpperCamelCase__				:			Optional[Any]=False  ,	UpperCamelCase__				:			int=64  ,	UpperCamelCase__				:			Tuple=0.1      )						->			Any:






				"""simple docstring"""







				if self.sp_model_kwargs.get('''enable_sampling'''      ) is True:
						snake_case  :       Optional[int]     			=						True
				if self.sp_model_kwargs.get('''alpha'''      ) is not None:
						snake_case  :       Tuple     			=						self.sp_model_kwargs.get('''alpha'''      )
				if self.sp_model_kwargs.get('''nbest_size'''      ) is not None:
						snake_case  :       List[str]     			=						self.sp_model_kwargs.get('''nbest_size'''      )

				if not enable_sampling:
						snake_case  :       List[Any]     			=						self.sp_model.EncodeAsPieces(UpperCamelCase__      )
				else:
						snake_case  :       Optional[int]     			=						self.sp_model.SampleEncodeAsPieces(UpperCamelCase__  ,	UpperCamelCase__  ,	UpperCamelCase__      )
				snake_case  :       Dict     			=						[]
				for pi, piece in enumerate(UpperCamelCase__      ):
						if piece == SPIECE_UNDERLINE:
								if not pieces[pi + 1].startswith(UpperCamelCase__      ) and pi != 0:
										new_pieces.append(UpperCamelCase__      )
										continue
								else:
										continue
						snake_case  :       Union[str, Any]     			=						0
						for i, chunk in enumerate(UpperCamelCase__      ):
								if chunk == SPIECE_UNDERLINE:
										continue
								if self.is_ch_char(UpperCamelCase__      ) or self.is_punct(UpperCamelCase__      ):
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
												new_pieces.append(piece[lst_i:i]      )
										new_pieces.append(UpperCamelCase__      )
										snake_case  :       Dict     			=						i + 1
								elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
												new_pieces.append(piece[lst_i:i]      )
										snake_case  :       str     			=						i
								elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
												new_pieces.append(piece[lst_i:i]      )
										snake_case  :       int     			=						i
						if len(UpperCamelCase__      ) > lst_i:
								new_pieces.append(piece[lst_i:]      )
				return new_pieces



		def       lowerCAmelCase				(      self				:			Union[str, Any]  ,	UpperCamelCase__				:			Tuple      )						->			Any:






				"""simple docstring"""







				snake_case  :       str     			=						''''''.join(UpperCamelCase__      ).replace(UpperCamelCase__  ,	''' '''      ).strip()
				return out_string



		def       lowerCAmelCase				(      self				:			str  ,	UpperCamelCase__				:			Any      )						->			List[Any]:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						self.convert_ids_to_tokens(UpperCamelCase__      )
				snake_case  :       Dict     			=						''''''.join(UpperCamelCase__      ).replace(UpperCamelCase__  ,	''' '''      ).strip()
				return out_string



		def       lowerCAmelCase				(      self				:			str  ,	UpperCamelCase__				:			Tuple      )						->			Any:






				"""simple docstring"""







				return self.vocab.get(UpperCamelCase__  ,	self.vocab.get(self.unk_token      )      )



		def       lowerCAmelCase				(      self				:			Any  ,	UpperCamelCase__				:			Union[str, Any]      )						->			Tuple:






				"""simple docstring"""







				return self.reverse_vocab.get(UpperCamelCase__  ,	self.unk_token      )



		def       lowerCAmelCase				(      self				:			Optional[Any]  ,	UpperCamelCase__				:			List[str]  ,	UpperCamelCase__				:			Optional[Any]=None      )						->			Optional[int]:






				"""simple docstring"""







				if token_ids_a is None:
						return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				snake_case  :       Tuple     			=						[self.cls_token_id]
				snake_case  :       Tuple     			=						[self.sep_token_id]
				return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



		def       lowerCAmelCase				(      self				:			Any  ,	UpperCamelCase__				:			List[Any]  ,	UpperCamelCase__				:			List[Any]=None      )						->			str:






				"""simple docstring"""







				if offset_mapping_a is None:
						return [(0, 0)] + offset_mapping_a + [(0, 0)]

				return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



		def       lowerCAmelCase				(      self				:			Dict  ,	UpperCamelCase__				:			List[str]  ,	UpperCamelCase__				:			Dict=None  ,	UpperCamelCase__				:			List[str]=False      )						->			Optional[int]:






				"""simple docstring"""







				if already_has_special_tokens:
						if token_ids_a is not None:
								raise ValueError(
								    '''You should not supply a second sequence if the provided sequence of '''
								    '''ids is already formatted with special tokens for the model.'''      )
						return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

				if token_ids_a is not None:
						return [1] + ([0] * len(UpperCamelCase__      )) + [1, 1] + ([0] * len(UpperCamelCase__      )) + [1]
				return [1] + ([0] * len(UpperCamelCase__      )) + [1]



		def       lowerCAmelCase				(      self				:			List[str]  ,	UpperCamelCase__				:			List[int]  ,	UpperCamelCase__				:			Optional[List[int]] = None      )						->			List[int]:






				"""simple docstring"""







				if token_ids_a is None:
						# [CLS] X [SEP]
						return (len(UpperCamelCase__      ) + 2) * [0]

				# [CLS] A [SEP] [SEP] B [SEP]
				return [0] * (len(UpperCamelCase__      ) + 1) + [1] * (len(UpperCamelCase__      ) + 3)



		def       lowerCAmelCase				(      self				:			Optional[int]  ,	UpperCamelCase__				:			Any      )						->			List[str]:






				"""simple docstring"""







				if "\u4e00" <= char <= "\u9fff":
						return True
				return False



		def       lowerCAmelCase				(      self				:			Optional[int]  ,	UpperCamelCase__				:			Tuple      )						->			str:






				"""simple docstring"""







				if ("a" <= char <= "z") or ("A" <= char <= "Z"):
						return True
				return False



		def       lowerCAmelCase				(      self				:			Optional[Any]  ,	UpperCamelCase__				:			str      )						->			int:






				"""simple docstring"""







				if char in ",;:.?!~，；：。？！《》【】":
						return True
				return False



		def       lowerCAmelCase				(      self				:			List[Any]  ,	UpperCamelCase__				:			Union[str, Any]      )						->			int:






				"""simple docstring"""







				if char == " " or char == "\t" or char == "\n" or char == "\r":
						return True
				if len(UpperCamelCase__      ) == 1:
						snake_case  :       Dict     			=						unicodedata.category(UpperCamelCase__      )
						if cat == "Zs":
								return True
				return False



		def       lowerCAmelCase				(      self				:			Tuple  ,	UpperCamelCase__				:			List[Any]      )						->			Optional[int]:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						{}
				with io.open(UpperCamelCase__  ,	'''r'''  ,	encoding='''utf-8'''      ) as f:
						for index, line in enumerate(UpperCamelCase__      ):
								snake_case  :       int     			=						line.rstrip('''\n'''      )
								snake_case  :       str     			=						int(UpperCamelCase__      )

				return token_to_idx



		def       lowerCAmelCase				(      self				:			Optional[Any]  ,	UpperCamelCase__				:			str  ,	UpperCamelCase__				:			Optional[str] = None      )						->			Tuple[str]:






				"""simple docstring"""







				snake_case  :       Optional[int]     			=						0
				if os.path.isdir(UpperCamelCase__      ):
						snake_case  :       int     			=						os.path.join(
						    UpperCamelCase__  ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
				else:
						snake_case  :       Optional[Any]     			=						(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
				with open(UpperCamelCase__  ,	'''w'''  ,	encoding='''utf-8'''      ) as writer:
						for token, token_index in sorted(self.vocab.items()  ,	key=lambda UpperCamelCase__      : kv[1]      ):
								if index != token_index:
										logger.warning(
										    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
										    ''' Please check that the vocabulary is not corrupted!'''      )
										snake_case  :       Dict     			=						token_index
								writer.write(token + '''\n'''      )
								index += 1

				snake_case  :       List[Any]     			=						os.path.join(UpperCamelCase__  ,	'''sentencepiece.bpe.model'''      )
				with open(UpperCamelCase__  ,	'''wb'''      ) as fi:
						snake_case  :       Tuple     			=						self.sp_model.serialized_model_proto()
						fi.write(UpperCamelCase__      )

				return (vocab_file,)




'''simple docstring'''
import os
from collections.abc import Iterator




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ = "."			)  ->							Iterator[str]:


		'''simple docstring'''


		for dir_path, dir_names, filenames in os.walk(SCREAMING_SNAKE_CASE__			):
				snake_case  :       Optional[Any]     			=						[d for d in dir_names if d != '''scripts''' and d[0] not in '''._''']
				for filename in filenames:
						if filename == "__init__.py":
								continue
						if os.path.splitext(SCREAMING_SNAKE_CASE__			)[1] in (".py", ".ipynb"):
								yield os.path.join(SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			).lstrip('''./'''			)




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__			)  ->							List[str]:


		'''simple docstring'''


		return F'{i * "  "}*' if i else "\n##"




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)  ->							str:


		'''simple docstring'''


		snake_case  :       Union[str, Any]     			=						old_path.split(os.sep			)
		for i, new_part in enumerate(new_path.split(os.sep			)			):
				if (i + 1 > len(SCREAMING_SNAKE_CASE__			) or old_parts[i] != new_part) and new_part:
						print(F'{md_prefix(SCREAMING_SNAKE_CASE__			)} {new_part.replace("_" ,       " "			).title()}'			)
		return new_path




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ = "."			)  ->							None:


		'''simple docstring'''


		snake_case  :       List[Any]     			=						''''''
		for filepath in sorted(good_file_paths(SCREAMING_SNAKE_CASE__			)			):
				snake_case			,snake_case  :       Optional[Any]     			=						os.path.split(SCREAMING_SNAKE_CASE__			)
				if filepath != old_path:
						snake_case  :       Dict     			=						print_path(SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)
				snake_case  :       int     			=						(filepath.count(os.sep			) + 1) if filepath else 0
				snake_case  :       int     			=						F'{filepath}/{filename}'.replace(''' ''' ,       '''%20'''			)
				snake_case  :       int     			=						os.path.splitext(filename.replace('''_''' ,       ''' '''			).title()			)[0]
				print(F'{md_prefix(SCREAMING_SNAKE_CASE__			)} [{filename}]({url})'			)


if __name__ == "__main__":
						print_directory_md(".")


