





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__           =			logging.get_logger(__name__)

UpperCamelCase__           =			"▁"

UpperCamelCase__           =			{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

UpperCamelCase__           =			{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

UpperCamelCase__           =			{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

UpperCamelCase__           =			{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

UpperCamelCase__           =			{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   List[str]      =							["input_ids"]

							snake_case  :   Dict      =							VOCAB_FILES_NAMES
							snake_case  :   Optional[int]      =							PRETRAINED_INIT_CONFIGURATION
							snake_case  :   Any      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case  :   Optional[Any]      =							PRETRAINED_VOCAB_FILES_MAP
							snake_case  :   List[str]      =							RESOURCE_FILES_NAMES






							def __init__(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=None	,			__lowerCAmelCase=False	,			__lowerCAmelCase="utf8"	,			__lowerCAmelCase="[UNK]"	,			__lowerCAmelCase="[SEP]"	,			__lowerCAmelCase="[PAD]"	,			__lowerCAmelCase="[CLS]"	,			__lowerCAmelCase="[MASK]"	,			__lowerCAmelCase = None	,			**__lowerCAmelCase	,			):
														# Mask token behave like a normal word, i.e. include the space before it and
														# is included in the raw text, there should be a match in a non-normalized sentence.

														UpperCamelCase__					  =      {} if sp_model_kwargs is None else sp_model_kwargs
														super().__init__(
														    do_lower_case=__lowerCAmelCase	,			unk_token=__lowerCAmelCase	,			sep_token=__lowerCAmelCase	,			pad_token=__lowerCAmelCase	,			cls_token=__lowerCAmelCase	,			mask_token=__lowerCAmelCase	,			vocab_file=__lowerCAmelCase	,			encoding=__lowerCAmelCase	,			sp_model_kwargs=self.sp_model_kwargs	,			**__lowerCAmelCase	,			)
														UpperCamelCase__					  =      do_lower_case
														UpperCamelCase__					  =      sentencepiece_model_ckpt
														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(__lowerCAmelCase	)

														# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
														if vocab_file is not None:
																					UpperCamelCase__					  =      self.load_vocab(filepath=__lowerCAmelCase	)
														else:
																					UpperCamelCase__					  =      {self.sp_model.id_to_piece(__lowerCAmelCase	): id for id in range(self.sp_model.get_piece_size()	)}
														UpperCamelCase__					  =      {v: k for k, v in self.vocab.items()}






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														if text is None:
																					return None

														UpperCamelCase__					  =      self.tokenize(__lowerCAmelCase	)
														UpperCamelCase__  ,  UpperCamelCase__					  =      """""", []

														for i, ch in enumerate(__lowerCAmelCase	):
																					if ch in self.SP_CHAR_MAPPING:
																												UpperCamelCase__					  =      self.SP_CHAR_MAPPING.get(__lowerCAmelCase	)
																					else:
																												UpperCamelCase__					  =      unicodedata.normalize("""NFKC"""	,			__lowerCAmelCase	)
																					if self.is_whitespace(__lowerCAmelCase	):
																												continue
																					normalized_text += ch
																					char_mapping.extend([i] * len(__lowerCAmelCase	)	)

														UpperCamelCase__  ,  UpperCamelCase__  ,  UpperCamelCase__					  =      normalized_text, [], 0

														if self.do_lower_case:
																					UpperCamelCase__					  =      text.lower()

														for token in split_tokens:
																					if token[:1] == "▁":
																												UpperCamelCase__					  =      token[1:]
																					UpperCamelCase__					  =      text[offset:].index(__lowerCAmelCase	) + offset
																					UpperCamelCase__					  =      start + len(__lowerCAmelCase	)

																					token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)	)
																					UpperCamelCase__					  =      end
														return token_mapping






							@property
							def 					_lowerCamelCase		(    self	):
														return len(self.vocab	)






							def 					_lowerCamelCase		(    self	):
														return dict(self.vocab	,			**self.added_tokens_encoder	)






							def __getstate__(    self	):
														UpperCamelCase__					  =      self.__dict__.copy()
														UpperCamelCase__					  =      None
														return state






							def __setstate__(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      d

														# for backward compatibility
														if not hasattr(self	,			"""sp_model_kwargs"""	):
																					UpperCamelCase__					  =      {}

														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(self.sentencepiece_model_ckpt	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return "".join((self.SP_CHAR_MAPPING.get(__lowerCAmelCase	,			__lowerCAmelCase	) for c in text)	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=64	,			__lowerCAmelCase=0.1	):

														if self.sp_model_kwargs.get("""enable_sampling"""	) is True:
																					UpperCamelCase__					  =      True
														if self.sp_model_kwargs.get("""alpha"""	) is not None:
																					UpperCamelCase__					  =      self.sp_model_kwargs.get("""alpha"""	)
														if self.sp_model_kwargs.get("""nbest_size"""	) is not None:
																					UpperCamelCase__					  =      self.sp_model_kwargs.get("""nbest_size"""	)

														if not enable_sampling:
																					UpperCamelCase__					  =      self.sp_model.EncodeAsPieces(__lowerCAmelCase	)
														else:
																					UpperCamelCase__					  =      self.sp_model.SampleEncodeAsPieces(__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase	)
														UpperCamelCase__					  =      []
														for pi, piece in enumerate(__lowerCAmelCase	):
																					if piece == SPIECE_UNDERLINE:
																												if not pieces[pi + 1].startswith(__lowerCAmelCase	) and pi != 0:
																																			new_pieces.append(__lowerCAmelCase	)
																																			continue
																												else:
																																			continue
																					UpperCamelCase__					  =      0
																					for i, chunk in enumerate(__lowerCAmelCase	):
																												if chunk == SPIECE_UNDERLINE:
																																			continue
																												if self.is_ch_char(__lowerCAmelCase	) or self.is_punct(__lowerCAmelCase	):
																																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																										new_pieces.append(piece[lst_i:i]	)
																																			new_pieces.append(__lowerCAmelCase	)
																																			UpperCamelCase__					  =      i + 1
																												elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																										new_pieces.append(piece[lst_i:i]	)
																																			UpperCamelCase__					  =      i
																												elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																										new_pieces.append(piece[lst_i:i]	)
																																			UpperCamelCase__					  =      i
																					if len(__lowerCAmelCase	) > lst_i:
																												new_pieces.append(piece[lst_i:]	)
														return new_pieces






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      """""".join(__lowerCAmelCase	).replace(__lowerCAmelCase	,			""" """	).strip()
														return out_string






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      self.convert_ids_to_tokens(__lowerCAmelCase	)
														UpperCamelCase__					  =      """""".join(__lowerCAmelCase	).replace(__lowerCAmelCase	,			""" """	).strip()
														return out_string






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.vocab.get(__lowerCAmelCase	,			self.vocab.get(self.unk_token	)	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.reverse_vocab.get(__lowerCAmelCase	,			self.unk_token	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=None	):
														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														UpperCamelCase__					  =      [self.cls_token_id]
														UpperCamelCase__					  =      [self.sep_token_id]
														return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=None	):
														if offset_mapping_a is None:
																					return [(0, 0)] + offset_mapping_a + [(0, 0)]

														return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=None	,			__lowerCAmelCase=False	):

														if already_has_special_tokens:
																					if token_ids_a is not None:
																												raise ValueError(
																												    """You should not supply a second sequence if the provided sequence of """
																												    """ids is already formatted with special tokens for the model."""	)
																					return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

														if token_ids_a is not None:
																					return [1] + ([0] * len(__lowerCAmelCase	)) + [1, 1] + ([0] * len(__lowerCAmelCase	)) + [1]
														return [1] + ([0] * len(__lowerCAmelCase	)) + [1]






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
														if token_ids_a is None:
																					# [CLS] X [SEP]
																					return (len(__lowerCAmelCase	) + 2) * [0]

														# [CLS] A [SEP] [SEP] B [SEP]
														return [0] * (len(__lowerCAmelCase	) + 1) + [1] * (len(__lowerCAmelCase	) + 3)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														if "\u4e00" <= char <= "\u9fff":
																					return True
														return False






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																					return True
														return False






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														if char in ",;:.?!~，；：。？！《》【】":
																					return True
														return False






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														if char == " " or char == "\t" or char == "\n" or char == "\r":
																					return True
														if len(__lowerCAmelCase	) == 1:
																					UpperCamelCase__					  =      unicodedata.category(__lowerCAmelCase	)
																					if cat == "Zs":
																												return True
														return False






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      {}
														with io.open(__lowerCAmelCase	,			"""r"""	,			encoding="""utf-8"""	) as f:
																					for index, line in enumerate(__lowerCAmelCase	):
																												UpperCamelCase__					  =      line.rstrip("""\n"""	)
																												UpperCamelCase__					  =      int(__lowerCAmelCase	)

														return token_to_idx







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      0
														if os.path.isdir(__lowerCAmelCase	):
																					UpperCamelCase__					  =      os.path.join(
																					    __lowerCAmelCase	,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)
														else:
																					UpperCamelCase__					  =      (filename_prefix + """-""" if filename_prefix else """""") + save_directory
														with open(__lowerCAmelCase	,			"""w"""	,			encoding="""utf-8"""	) as writer:
																					for token, token_index in sorted(self.vocab.items()	,			key=lambda __lowerCAmelCase	: kv[1]	):
																												if index != token_index:
																																			logger.warning(
																																			    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																																			    """ Please check that the vocabulary is not corrupted!"""	)
																																			UpperCamelCase__					  =      token_index
																												writer.write(token + """\n"""	)
																												index += 1

														UpperCamelCase__					  =      os.path.join(__lowerCAmelCase	,			"""sentencepiece.bpe.model"""	)
														with open(__lowerCAmelCase	,			"""wb"""	) as fi:
																					UpperCamelCase__					  =      self.sp_model.serialized_model_proto()
																					fi.write(__lowerCAmelCase	)

														return (vocab_file,)











import copy
import random

from transformers import CLIPTokenizer

class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def __init__(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														super().__init__(*__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      {}






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      super().add_tokens(__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	)
														if num_added_tokens == 0:
																					raise ValueError(
																					    f"""The tokenizer already contains the token {placeholder_token}. Please pass a different"""
																					    """ `placeholder_token` that is not already in the tokenizer."""	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			__lowerCAmelCase=1	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      []
														if num_vec_per_token == 1:
																					self.try_adding_tokens(__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	)
																					output.append(__lowerCAmelCase	)
														else:
																					UpperCamelCase__					  =      []
																					for i in range(__lowerCAmelCase	):
																												UpperCamelCase__					  =      placeholder_token + f"""_{i}"""
																												self.try_adding_tokens(__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	)
																												output.append(__lowerCAmelCase	)
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
														for token in self.token_map:
																					if token in placeholder_token:
																												raise ValueError(
																												    f"""The tokenizer already has placeholder token {token} that can get confused with"""
																												    f""" {placeholder_token}keep placeholder tokens independent"""	)
														UpperCamelCase__					  =      output






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=1.0	):
														if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	):
																					UpperCamelCase__					  =      []
																					for i in range(len(__lowerCAmelCase	)	):
																												output.append(self.replace_placeholder_tokens_in_text(text[i]	,			vector_shuffle=__lowerCAmelCase	)	)
																					return output
														for placeholder_token in self.token_map:
																					if placeholder_token in text:
																												UpperCamelCase__					  =      self.token_map[placeholder_token]
																												UpperCamelCase__					  =      tokens[: 1 + int(len(__lowerCAmelCase	) * prop_tokens_to_load	)]
																												if vector_shuffle:
																																			UpperCamelCase__					  =      copy.copy(__lowerCAmelCase	)
																																			random.shuffle(__lowerCAmelCase	)
																												UpperCamelCase__					  =      text.replace(__lowerCAmelCase	,			""" """.join(__lowerCAmelCase	)	)
														return text






							def __call__(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=1.0	,			**__lowerCAmelCase	):
														return super().__call__(
														    self.replace_placeholder_tokens_in_text(
														        __lowerCAmelCase	,			vector_shuffle=__lowerCAmelCase	,			prop_tokens_to_load=__lowerCAmelCase	)	,			*__lowerCAmelCase	,			**__lowerCAmelCase	,			)







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=1.0	,			**__lowerCAmelCase	):
														return super().encode(
														    self.replace_placeholder_tokens_in_text(
														        __lowerCAmelCase	,			vector_shuffle=__lowerCAmelCase	,			prop_tokens_to_load=__lowerCAmelCase	)	,			*__lowerCAmelCase	,			**__lowerCAmelCase	,			)




