



"""simple docstring"""


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A :				Any					    = logging.get_logger(__name__)

__A :				Tuple					    = '▁'

__A :				Optional[int]					    = {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

__A :				Dict					    = {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

__A :				Dict					    = {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

__A :				Dict					    = {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

__A :				Any					    = {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}
class       _a			(       lowerCAmelCase):
			"""simple docstring"""
			UpperCamelCase__         =						["""input_ids"""]

			UpperCamelCase__         =						VOCAB_FILES_NAMES
			UpperCamelCase__         =						PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase__         =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase__         =						PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase__         =						RESOURCE_FILES_NAMES



			def __init__(		self  :     Optional[int]					, __UpperCamelCase  :     Optional[Any]					, __UpperCamelCase  :     str=None					, __UpperCamelCase  :     str=False					, __UpperCamelCase  :     List[str]="utf8"					, __UpperCamelCase  :     Any="[UNK]"					, __UpperCamelCase  :     List[Any]="[SEP]"					, __UpperCamelCase  :     int="[PAD]"					, __UpperCamelCase  :     Union[str, Any]="[CLS]"					, __UpperCamelCase  :     List[Any]="[MASK]"					, __UpperCamelCase  :     List[str] = None					, **__UpperCamelCase  :     Tuple					, )->Optional[Any]:
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.

							_UpperCAmelCase      		=     {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=A_					, unk_token=A_					, sep_token=A_					, pad_token=A_					, cls_token=A_					, mask_token=A_					, vocab_file=A_					, encoding=A_					, sp_model_kwargs=self.sp_model_kwargs					, **A_					, )
							_UpperCAmelCase      		=     do_lower_case
							_UpperCAmelCase      		=     sentencepiece_model_ckpt
							_UpperCAmelCase      		=     spm.SentencePieceProcessor(**self.sp_model_kwargs )
							self.sp_model.Load(A_ )

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
											_UpperCAmelCase      		=     self.load_vocab(filepath=A_ )
							else:
											_UpperCAmelCase      		=     {self.sp_model.id_to_piece(A_ ): id for id in range(self.sp_model.get_piece_size() )}
							_UpperCAmelCase      		=     {v: k for k, v in self.vocab.items()}



			def 			lowercase__    (		self  :     str					, __UpperCamelCase  :     Optional[int] )->List[Any]:
							if text is None:
											return None

							_UpperCAmelCase      		=     self.tokenize(A_ )
							_UpperCAmelCase      		=     """""", []

							for i, ch in enumerate(A_ ):
											if ch in self.SP_CHAR_MAPPING:
															_UpperCAmelCase      		=     self.SP_CHAR_MAPPING.get(A_ )
											else:
															_UpperCAmelCase      		=     unicodedata.normalize('''NFKC'''					, A_ )
											if self.is_whitespace(A_ ):
															continue
											normalized_text += ch
											char_mapping.extend([i] * len(A_ ) )

							_UpperCAmelCase      		=     normalized_text, [], 0

							if self.do_lower_case:
											_UpperCAmelCase      		=     text.lower()

							for token in split_tokens:
											if token[:1] == "▁":
															_UpperCAmelCase      		=     token[1:]
											_UpperCAmelCase      		=     text[offset:].index(A_ ) + offset
											_UpperCAmelCase      		=     start + len(A_ )

											token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1) )
											_UpperCAmelCase      		=     end
							return token_mapping



			@property
			def 			lowercase__    (		self  :     str )->Union[str, Any]:
							return len(self.vocab )



			def 			lowercase__    (		self  :     Optional[Any] )->Dict:
							return dict(self.vocab					, **self.added_tokens_encoder )



			def __getstate__(		self  :     Union[str, Any] )->Tuple:
							_UpperCAmelCase      		=     self.__dict__.copy()
							_UpperCAmelCase      		=     None
							return state



			def __setstate__(		self  :     Union[str, Any]					, __UpperCamelCase  :     Optional[Any] )->Dict:
							_UpperCAmelCase      		=     d

							# for backward compatibility
							if not hasattr(self					, '''sp_model_kwargs''' ):
											_UpperCAmelCase      		=     {}

							_UpperCAmelCase      		=     spm.SentencePieceProcessor(**self.sp_model_kwargs )
							self.sp_model.Load(self.sentencepiece_model_ckpt )



			def 			lowercase__    (		self  :     Optional[Any]					, __UpperCamelCase  :     Optional[Any] )->Tuple:
							return "".join((self.SP_CHAR_MAPPING.get(A_					, A_ ) for c in text) )



			def 			lowercase__    (		self  :     Dict					, __UpperCamelCase  :     Optional[int]					, __UpperCamelCase  :     Optional[int]=False					, __UpperCamelCase  :     int=6_4					, __UpperCamelCase  :     str=0.1 )->Optional[Any]:

							if self.sp_model_kwargs.get('''enable_sampling''' ) is True:
											_UpperCAmelCase      		=     True
							if self.sp_model_kwargs.get('''alpha''' ) is not None:
											_UpperCAmelCase      		=     self.sp_model_kwargs.get('''alpha''' )
							if self.sp_model_kwargs.get('''nbest_size''' ) is not None:
											_UpperCAmelCase      		=     self.sp_model_kwargs.get('''nbest_size''' )

							if not enable_sampling:
											_UpperCAmelCase      		=     self.sp_model.EncodeAsPieces(A_ )
							else:
											_UpperCAmelCase      		=     self.sp_model.SampleEncodeAsPieces(A_					, A_					, A_ )
							_UpperCAmelCase      		=     []
							for pi, piece in enumerate(A_ ):
											if piece == SPIECE_UNDERLINE:
															if not pieces[pi + 1].startswith(A_ ) and pi != 0:
																			new_pieces.append(A_ )
																			continue
															else:
																			continue
											_UpperCAmelCase      		=     0
											for i, chunk in enumerate(A_ ):
															if chunk == SPIECE_UNDERLINE:
																			continue
															if self.is_ch_char(A_ ) or self.is_punct(A_ ):
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i] )
																			new_pieces.append(A_ )
																			_UpperCAmelCase      		=     i + 1
															elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i] )
																			_UpperCAmelCase      		=     i
															elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i] )
																			_UpperCAmelCase      		=     i
											if len(A_ ) > lst_i:
															new_pieces.append(piece[lst_i:] )
							return new_pieces



			def 			lowercase__    (		self  :     Any					, __UpperCamelCase  :     Dict )->List[Any]:
							_UpperCAmelCase      		=     """""".join(A_ ).replace(A_					, ''' ''' ).strip()
							return out_string



			def 			lowercase__    (		self  :     str					, __UpperCamelCase  :     Tuple )->str:
							_UpperCAmelCase      		=     self.convert_ids_to_tokens(A_ )
							_UpperCAmelCase      		=     """""".join(A_ ).replace(A_					, ''' ''' ).strip()
							return out_string



			def 			lowercase__    (		self  :     Optional[int]					, __UpperCamelCase  :     str )->Optional[int]:
							return self.vocab.get(A_					, self.vocab.get(self.unk_token ) )



			def 			lowercase__    (		self  :     List[Any]					, __UpperCamelCase  :     Optional[Any] )->str:
							return self.reverse_vocab.get(A_					, self.unk_token )



			def 			lowercase__    (		self  :     str					, __UpperCamelCase  :     Dict					, __UpperCamelCase  :     Optional[Any]=None )->List[Any]:
							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_UpperCAmelCase      		=     [self.cls_token_id]
							_UpperCAmelCase      		=     [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



			def 			lowercase__    (		self  :     Tuple					, __UpperCamelCase  :     List[Any]					, __UpperCamelCase  :     int=None )->int:
							if offset_mapping_a is None:
											return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



			def 			lowercase__    (		self  :     str					, __UpperCamelCase  :     Union[str, Any]					, __UpperCamelCase  :     List[str]=None					, __UpperCamelCase  :     Optional[int]=False )->Any:

							if already_has_special_tokens:
											if token_ids_a is not None:
															raise ValueError(
															    '''You should not supply a second sequence if the provided sequence of '''
															    '''ids is already formatted with special tokens for the model.''' )
											return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
											return [1] + ([0] * len(A_ )) + [1, 1] + ([0] * len(A_ )) + [1]
							return [1] + ([0] * len(A_ )) + [1]



			def 			lowercase__    (		self  :     List[str]					, __UpperCamelCase  :     List[str]					, __UpperCamelCase  :     str = None )->Union[str, Any]:
							# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
							if token_ids_a is None:
											# [CLS] X [SEP]
											return (len(A_ ) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(A_ ) + 1) + [1] * (len(A_ ) + 3)



			def 			lowercase__    (		self  :     List[Any]					, __UpperCamelCase  :     Optional[int] )->Union[str, Any]:
							if "\u4e00" <= char <= "\u9fff":
											return True
							return False



			def 			lowercase__    (		self  :     List[str]					, __UpperCamelCase  :     Dict )->List[str]:
							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
											return True
							return False



			def 			lowercase__    (		self  :     Optional[int]					, __UpperCamelCase  :     List[str] )->Tuple:
							if char in ",;:.?!~，；：。？！《》【】":
											return True
							return False



			def 			lowercase__    (		self  :     Union[str, Any]					, __UpperCamelCase  :     int )->str:
							if char == " " or char == "\t" or char == "\n" or char == "\r":
											return True
							if len(A_ ) == 1:
											_UpperCAmelCase      		=     unicodedata.category(A_ )
											if cat == "Zs":
															return True
							return False



			def 			lowercase__    (		self  :     Union[str, Any]					, __UpperCamelCase  :     Optional[Any] )->Dict:
							_UpperCAmelCase      		=     {}
							with io.open(A_					, '''r'''					, encoding='''utf-8''' ) as f:
											for index, line in enumerate(A_ ):
															_UpperCAmelCase      		=     line.rstrip('''\n''' )
															_UpperCAmelCase      		=     int(A_ )

							return token_to_idx



			def 			lowercase__    (		self  :     Optional[Any]					, __UpperCamelCase  :     Optional[int]					, __UpperCamelCase  :     Union[str, Any] = None )->int:
							_UpperCAmelCase      		=     0
							if os.path.isdir(A_ ):
											_UpperCAmelCase      		=     os.path.join(
											    A_					, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
							else:
											_UpperCAmelCase      		=     (filename_prefix + """-""" if filename_prefix else """""") + save_directory
							with open(A_					, '''w'''					, encoding='''utf-8''' ) as writer:
											for token, token_index in sorted(self.vocab.items()					, key=lambda __UpperCamelCase : kv[1] ):
															if index != token_index:
																			logger.warning(
																			    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																			    ''' Please check that the vocabulary is not corrupted!''' )
																			_UpperCAmelCase      		=     token_index
															writer.write(token + '''\n''' )
															index += 1

							_UpperCAmelCase      		=     os.path.join(A_					, '''sentencepiece.bpe.model''' )
							with open(A_					, '''wb''' ) as fi:
											_UpperCAmelCase      		=     self.sp_model.serialized_model_proto()
											fi.write(A_ )

							return (vocab_file,)







def    a__    (		snake_case__					:							int						,   snake_case__					:							int     ):
  return 1 if input_a == input_a else 0




def    a__    (		):
  assert xnor_gate(0						,   0     ) == 1
  assert xnor_gate(0						,   1     ) == 0
  assert xnor_gate(1						,   0     ) == 0
  assert xnor_gate(1						,   1     ) == 1


if __name__ == "__main__":
 print(xnor_gate(0, 0))
 print(xnor_gate(0, 1))
 print(xnor_gate(1, 0))
 print(xnor_gate(1, 1))





