






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__				     =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__				     =			'''▁'''

SCREAMING_SNAKE_CASE__				     =			{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

SCREAMING_SNAKE_CASE__				     =			{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

SCREAMING_SNAKE_CASE__				     =			{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

SCREAMING_SNAKE_CASE__				     =			{
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

SCREAMING_SNAKE_CASE__				     =			{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}






class 			_UpperCamelCase(       __lowerCamelCase		):
		__SCREAMING_SNAKE_CASE  :       List[str]							=							["input_ids"]

		__SCREAMING_SNAKE_CASE  :       Optional[Any]							=							VOCAB_FILES_NAMES
		__SCREAMING_SNAKE_CASE  :       Optional[Any]							=							PRETRAINED_INIT_CONFIGURATION
		__SCREAMING_SNAKE_CASE  :       Dict							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__SCREAMING_SNAKE_CASE  :       List[Any]							=							PRETRAINED_VOCAB_FILES_MAP
		__SCREAMING_SNAKE_CASE  :       int							=							RESOURCE_FILES_NAMES

		def __init__(					self		:    Dict    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    Any=None    ,       SCREAMING_SNAKE_CASE__		:    List[str]=False    ,       SCREAMING_SNAKE_CASE__		:    List[str]="utf8"    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]="[UNK]"    ,       SCREAMING_SNAKE_CASE__		:    List[str]="[SEP]"    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]="[PAD]"    ,       SCREAMING_SNAKE_CASE__		:    List[str]="[CLS]"    ,       SCREAMING_SNAKE_CASE__		:    Tuple="[MASK]"    ,       SCREAMING_SNAKE_CASE__		:    Optional[Dict[str, Any]] = None    ,       **SCREAMING_SNAKE_CASE__		:    List[Any]    ,       ):



					'''simple docstring'''



					__a						:  Tuple									=     {} if sp_model_kwargs is None else sp_model_kwargs
					super().__init__(
					    do_lower_case=SCREAMING_SNAKE_CASE__    ,       unk_token=SCREAMING_SNAKE_CASE__    ,       sep_token=SCREAMING_SNAKE_CASE__    ,       pad_token=SCREAMING_SNAKE_CASE__    ,       cls_token=SCREAMING_SNAKE_CASE__    ,       mask_token=SCREAMING_SNAKE_CASE__    ,       vocab_file=SCREAMING_SNAKE_CASE__    ,       encoding=SCREAMING_SNAKE_CASE__    ,       sp_model_kwargs=self.sp_model_kwargs    ,       **SCREAMING_SNAKE_CASE__    ,       )
					__a						:  Optional[int]									=     do_lower_case
					__a						:  List[str]									=     sentencepiece_model_ckpt
					__a						:  Optional[Any]									=     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
					self.sp_model.Load(SCREAMING_SNAKE_CASE__    )

					# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
					if vocab_file is not None:
								__a						:  Optional[Any]									=     self.load_vocab(filepath=SCREAMING_SNAKE_CASE__    )
					else:
								__a						:  int									=     {self.sp_model.id_to_piece(SCREAMING_SNAKE_CASE__    ): id for id in range(self.sp_model.get_piece_size()    )}
					__a						:  str									=     {v: k for k, v in self.vocab.items()}

		def    __lowerCAmelCase  (					self		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ):



					'''simple docstring'''



					if text is None:
								return None

					__a						:  List[Any]									=     self.tokenize(SCREAMING_SNAKE_CASE__    )
					__a						,					__a						:  Union[str, Any]									=     '', []

					for i, ch in enumerate(SCREAMING_SNAKE_CASE__    ):
								if ch in self.SP_CHAR_MAPPING:
											__a						:  List[str]									=     self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE__    )
								else:
											__a						:  Any									=     unicodedata.normalize('NFKC'    ,       SCREAMING_SNAKE_CASE__    )
								if self.is_whitespace(SCREAMING_SNAKE_CASE__    ):
											continue
								normalized_text += ch
								char_mapping.extend([i] * len(SCREAMING_SNAKE_CASE__    )    )

					__a						,					__a						,					__a						:  str									=     normalized_text, [], 0

					if self.do_lower_case:
								__a						:  Tuple									=     text.lower()

					for token in split_tokens:
								if token[:1] == "▁":
											__a						:  List[Any]									=     token[1:]
								__a						:  Optional[Any]									=     text[offset:].index(SCREAMING_SNAKE_CASE__    ) + offset
								__a						:  List[Any]									=     start + len(SCREAMING_SNAKE_CASE__    )

								token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)    )
								__a						:  Tuple									=     end
					return token_mapping

		@property
		def    __lowerCAmelCase  (					self		:    Dict    ):



					'''simple docstring'''



					return len(self.vocab    )

		def    __lowerCAmelCase  (					self		:    List[Any]    ):



					'''simple docstring'''



					return dict(self.vocab    ,       **self.added_tokens_encoder    )

		def __getstate__(					self		:    Tuple    ):



					'''simple docstring'''



					__a						:  Tuple									=     self.__dict__.copy()
					__a						:  Any									=     None
					return state

		def __setstate__(					self		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    List[str]    ):



					'''simple docstring'''



					__a						:  Optional[Any]									=     d

					# for backward compatibility
					if not hasattr(self    ,       'sp_model_kwargs'    ):
								__a						:  str									=     {}

					__a						:  Optional[int]									=     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
					self.sp_model.Load(self.sentencepiece_model_ckpt    )

		def    __lowerCAmelCase  (					self		:    Optional[Any]    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					return "".join((self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) for c in text)    )

		def    __lowerCAmelCase  (					self		:    int    ,       SCREAMING_SNAKE_CASE__		:    List[str]    ,       SCREAMING_SNAKE_CASE__		:    int=False    ,       SCREAMING_SNAKE_CASE__		:    Tuple=6_4    ,       SCREAMING_SNAKE_CASE__		:    List[str]=0.1    ):



					'''simple docstring'''



					if self.sp_model_kwargs.get('enable_sampling'    ) is True:
								__a						:  Dict									=     True
					if self.sp_model_kwargs.get('alpha'    ) is not None:
								__a						:  int									=     self.sp_model_kwargs.get('alpha'    )
					if self.sp_model_kwargs.get('nbest_size'    ) is not None:
								__a						:  Any									=     self.sp_model_kwargs.get('nbest_size'    )

					if not enable_sampling:
								__a						:  int									=     self.sp_model.EncodeAsPieces(SCREAMING_SNAKE_CASE__    )
					else:
								__a						:  Optional[Any]									=     self.sp_model.SampleEncodeAsPieces(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )
					__a						:  str									=     []
					for pi, piece in enumerate(SCREAMING_SNAKE_CASE__    ):
								if piece == SPIECE_UNDERLINE:
											if not pieces[pi + 1].startswith(SCREAMING_SNAKE_CASE__    ) and pi != 0:
														new_pieces.append(SCREAMING_SNAKE_CASE__    )
														continue
											else:
														continue
								__a						:  Optional[int]									=     0
								for i, chunk in enumerate(SCREAMING_SNAKE_CASE__    ):
											if chunk == SPIECE_UNDERLINE:
														continue
											if self.is_ch_char(SCREAMING_SNAKE_CASE__    ) or self.is_punct(SCREAMING_SNAKE_CASE__    ):
														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																	new_pieces.append(piece[lst_i:i]    )
														new_pieces.append(SCREAMING_SNAKE_CASE__    )
														__a						:  Union[str, Any]									=     i + 1
											elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																	new_pieces.append(piece[lst_i:i]    )
														__a						:  int									=     i
											elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																	new_pieces.append(piece[lst_i:i]    )
														__a						:  Dict									=     i
								if len(SCREAMING_SNAKE_CASE__    ) > lst_i:
											new_pieces.append(piece[lst_i:]    )
					return new_pieces

		def    __lowerCAmelCase  (					self		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ):



					'''simple docstring'''



					__a						:  int									=     ''.join(SCREAMING_SNAKE_CASE__    ).replace(SCREAMING_SNAKE_CASE__    ,       ' '    ).strip()
					return out_string

		def    __lowerCAmelCase  (					self		:    Dict    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					__a						:  str									=     self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__    )
					__a						:  Optional[Any]									=     ''.join(SCREAMING_SNAKE_CASE__    ).replace(SCREAMING_SNAKE_CASE__    ,       ' '    ).strip()
					return out_string

		def    __lowerCAmelCase  (					self		:    Any    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]    ):



					'''simple docstring'''



					return self.vocab.get(SCREAMING_SNAKE_CASE__    ,       self.vocab.get(self.unk_token    )    )

		def    __lowerCAmelCase  (					self		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					return self.reverse_vocab.get(SCREAMING_SNAKE_CASE__    ,       self.unk_token    )

		def    __lowerCAmelCase  (					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    Tuple    ,       SCREAMING_SNAKE_CASE__		:    List[Any]=None    ):



					'''simple docstring'''



					if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					__a						:  Union[str, Any]									=     [self.cls_token_id]
					__a						:  Union[str, Any]									=     [self.sep_token_id]
					return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

		def    __lowerCAmelCase  (					self		:    Dict    ,       SCREAMING_SNAKE_CASE__		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    str=None    ):



					'''simple docstring'''



					if offset_mapping_a is None:
								return [(0, 0)] + offset_mapping_a + [(0, 0)]

					return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

		def    __lowerCAmelCase  (					self		:    Tuple    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    List[Any]=None    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]=False    ):



					'''simple docstring'''



					if already_has_special_tokens:
								if token_ids_a is not None:
											raise ValueError(
											    'You should not supply a second sequence if the provided sequence of '
											    'ids is already formatted with special tokens for the model.'    )
								return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

					if token_ids_a is not None:
								return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]
					return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]

		def    __lowerCAmelCase  (					self		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    List[int]    ,       SCREAMING_SNAKE_CASE__		:    Optional[List[int]] = None    ):



					'''simple docstring'''



					if token_ids_a is None:
								# [CLS] X [SEP]
								return (len(SCREAMING_SNAKE_CASE__    ) + 2) * [0]

					# [CLS] A [SEP] [SEP] B [SEP]
					return [0] * (len(SCREAMING_SNAKE_CASE__    ) + 1) + [1] * (len(SCREAMING_SNAKE_CASE__    ) + 3)

		def    __lowerCAmelCase  (					self		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					if "\u4e00" <= char <= "\u9fff":
								return True
					return False

		def    __lowerCAmelCase  (					self		:    Union[str, Any]    ,       SCREAMING_SNAKE_CASE__		:    str    ):



					'''simple docstring'''



					if ("a" <= char <= "z") or ("A" <= char <= "Z"):
								return True
					return False

		def    __lowerCAmelCase  (					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    Any    ):



					'''simple docstring'''



					if char in ",;:.?!~，；：。？！《》【】":
								return True
					return False

		def    __lowerCAmelCase  (					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					if char == " " or char == "\t" or char == "\n" or char == "\r":
								return True
					if len(SCREAMING_SNAKE_CASE__    ) == 1:
								__a						:  str									=     unicodedata.category(SCREAMING_SNAKE_CASE__    )
								if cat == "Zs":
											return True
					return False

		def    __lowerCAmelCase  (					self		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ):



					'''simple docstring'''



					__a						:  Union[str, Any]									=     {}
					with io.open(SCREAMING_SNAKE_CASE__    ,       'r'    ,       encoding='utf-8'    ) as f:
								for index, line in enumerate(SCREAMING_SNAKE_CASE__    ):
											__a						:  Optional[Any]									=     line.rstrip('\n'    )
											__a						:  Dict									=     int(SCREAMING_SNAKE_CASE__    )

					return token_to_idx



		def    __lowerCAmelCase  (					self		:    Any    ,       SCREAMING_SNAKE_CASE__		:    str    ,       SCREAMING_SNAKE_CASE__		:    Optional[str] = None    ):



					'''simple docstring'''



					__a						:  List[Any]									=     0
					if os.path.isdir(SCREAMING_SNAKE_CASE__    ):
								__a						:  Union[str, Any]									=     os.path.join(
								    SCREAMING_SNAKE_CASE__    ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
					else:
								__a						:  Union[str, Any]									=     (filename_prefix + '-' if filename_prefix else '') + save_directory
					with open(SCREAMING_SNAKE_CASE__    ,       'w'    ,       encoding='utf-8'    ) as writer:
								for token, token_index in sorted(self.vocab.items()    ,       key=lambda SCREAMING_SNAKE_CASE__    : kv[1]    ):
											if index != token_index:
														logger.warning(
														    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
														    ' Please check that the vocabulary is not corrupted!'    )
														__a						:  List[Any]									=     token_index
											writer.write(token + '\n'    )
											index += 1

					__a						:  List[Any]									=     os.path.join(SCREAMING_SNAKE_CASE__    ,       'sentencepiece.bpe.model'    )
					with open(SCREAMING_SNAKE_CASE__    ,       'wb'    ) as fi:
								__a						:  str									=     self.sp_model.serialized_model_proto()
								fi.write(SCREAMING_SNAKE_CASE__    )

					return (vocab_file,)












# Author: OMKAR PATHAK, Nwachukwu Chidiebere

# Use a Python dictionary to construct the graph.
from __future__ import annotations

from pprint import pformat
from typing import Generic, TypeVar

__lowerCamelCase			:	str					    =	TypeVar("""T""")



class    A__  (			Generic[T]     ):





      def __init__(					self			,				A_ = True  ):
       '''simple docstring'''




       UpperCamelCase      :   dict[T, list[T]]			      =  {}  # dictionary of lists
       UpperCamelCase      :   Union[str, Any]			      =  directed





      def 			__UpperCamelCase(					self			,				A_			,				A_  ):
       '''simple docstring'''




       if not self.directed:  # For undirected graphs
        # if both source vertex and destination vertex are both present in the
        # adjacency list, add destination vertex to source vertex list of adjacent
        # vertices and add source vertex to destination vertex list of adjacent
        # vertices.
        if source_vertex in self.adj_list and destination_vertex in self.adj_list:
         self.adj_list[source_vertex].append(A_  )
         self.adj_list[destination_vertex].append(A_  )
        # if only source vertex is present in adjacency list, add destination vertex
        # to source vertex list of adjacent vertices, then create a new vertex with
        # destination vertex as key and assign a list containing the source vertex
        # as it's first adjacent vertex.
        elif source_vertex in self.adj_list:
         self.adj_list[source_vertex].append(A_  )
         UpperCamelCase      :   Any			      =  [source_vertex]
        # if only destination vertex is present in adjacency list, add source vertex
        # to destination vertex list of adjacent vertices, then create a new vertex
        # with source vertex as key and assign a list containing the source vertex
        # as it's first adjacent vertex.
        elif destination_vertex in self.adj_list:
         self.adj_list[destination_vertex].append(A_  )
         UpperCamelCase      :   Any			      =  [destination_vertex]
        # if both source vertex and destination vertex are not present in adjacency
        # list, create a new vertex with source vertex as key and assign a list
        # containing the destination vertex as it's first adjacent vertex also
        # create a new vertex with destination vertex as key and assign a list
        # containing the source vertex as it's first adjacent vertex.
        else:
         UpperCamelCase      :   Union[str, Any]			      =  [destination_vertex]
         UpperCamelCase      :   List[str]			      =  [source_vertex]
       else:  # For directed graphs
        # if both source vertex and destination vertex are present in adjacency
        # list, add destination vertex to source vertex list of adjacent vertices.
        if source_vertex in self.adj_list and destination_vertex in self.adj_list:
         self.adj_list[source_vertex].append(A_  )
        # if only source vertex is present in adjacency list, add destination
        # vertex to source vertex list of adjacent vertices and create a new vertex
        # with destination vertex as key, which has no adjacent vertex
        elif source_vertex in self.adj_list:
         self.adj_list[source_vertex].append(A_  )
         UpperCamelCase      :   List[str]			      =  []
        # if only destination vertex is present in adjacency list, create a new
        # vertex with source vertex as key and assign a list containing destination
        # vertex as first adjacent vertex
        elif destination_vertex in self.adj_list:
         UpperCamelCase      :   Union[str, Any]			      =  [destination_vertex]
        # if both source vertex and destination vertex are not present in adjacency
        # list, create a new vertex with source vertex as key and a list containing
        # destination vertex as it's first adjacent vertex. Then create a new vertex
        # with destination vertex as key, which has no adjacent vertex
        else:
         UpperCamelCase      :   Union[str, Any]			      =  [destination_vertex]
         UpperCamelCase      :   Dict			      =  []

       return self





      def __repr__(					self  ):
       '''simple docstring'''




       return pformat(self.adj_list  )
