



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCAmelCase__						:	Dict					=logging.get_logger(__name__)

lowerCAmelCase__						:	Any					={"vocab_file": "spiece.model"}

lowerCAmelCase__						:	Union[str, Any]					={
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

lowerCAmelCase__						:	Union[str, Any]					={
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
lowerCAmelCase__						:	Union[str, Any]					=0
lowerCAmelCase__						:	List[Any]					=1
lowerCAmelCase__						:	List[Any]					=2
lowerCAmelCase__						:	Optional[int]					=3
lowerCAmelCase__						:	List[Any]					=4

class   UpperCAmelCase_    (  lowerCAmelCase__			):

						'''simple docstring'''




						UpperCamelCase__     :					Union[str, Any]							    =    VOCAB_FILES_NAMES
						UpperCamelCase__     :					Optional[Any]							    =    PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase__     :					Dict							    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase__     :					Optional[int]							    =    """left"""






						def __init__(       self ,				_A ,				_A=False ,				_A=True ,				_A=False ,				_A="<s>" ,				_A="</s>" ,				_A="<unk>" ,				_A="<sep>" ,				_A="<pad>" ,				_A="<cls>" ,				_A="<mask>" ,				_A=["<eop>", "<eod>"] ,				_A = None ,				**_A ,				):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   AddedToken(__a ,				lstrip=__a ,				rstrip=__a						) if isinstance(__a ,				__a						) else mask_token

													__SCREAMING_SNAKE_CASE										=   {} if sp_model_kwargs is None else sp_model_kwargs

													super().__init__(
													    do_lower_case=__a ,				remove_space=__a ,				keep_accents=__a ,				bos_token=__a ,				eos_token=__a ,				unk_token=__a ,				sep_token=__a ,				pad_token=__a ,				cls_token=__a ,				mask_token=__a ,				additional_special_tokens=__a ,				sp_model_kwargs=self.sp_model_kwargs ,				**__a ,				)

													__SCREAMING_SNAKE_CASE										=   3

													__SCREAMING_SNAKE_CASE										=   do_lower_case
													__SCREAMING_SNAKE_CASE										=   remove_space
													__SCREAMING_SNAKE_CASE										=   keep_accents
													__SCREAMING_SNAKE_CASE										=   vocab_file

													__SCREAMING_SNAKE_CASE										=   spm.SentencePieceProcessor(**self.sp_model_kwargs						)
													self.sp_model.Load(__a						)






						@property
						def 			_A      (       self						):







													'''simple docstring'''

													return len(self.sp_model						)






						def 			_A      (       self						):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   {self.convert_ids_to_tokens(__a						): i for i in range(self.vocab_size						)}
													vocab.update(self.added_tokens_encoder						)
													return vocab






						def __getstate__(       self						):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   self.__dict__.copy()
													__SCREAMING_SNAKE_CASE										=   None
													return state






						def __setstate__(       self ,				_A						):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   d

													# for backward compatibility
													if not hasattr(self ,				'sp_model_kwargs'						):
																				__SCREAMING_SNAKE_CASE										=   {}

													__SCREAMING_SNAKE_CASE										=   spm.SentencePieceProcessor(**self.sp_model_kwargs						)
													self.sp_model.Load(self.vocab_file						)






						def 			_A      (       self ,				_A						):







													'''simple docstring'''

													if self.remove_space:
																				__SCREAMING_SNAKE_CASE										=   ' '.join(inputs.strip().split()						)
													else:
																				__SCREAMING_SNAKE_CASE										=   inputs
													__SCREAMING_SNAKE_CASE										=   outputs.replace('``' ,				'\"'						).replace('\'\'' ,				'\"'						)

													if not self.keep_accents:
																				__SCREAMING_SNAKE_CASE										=   unicodedata.normalize('NFKD' ,				__a						)
																				__SCREAMING_SNAKE_CASE										=   ''.join([c for c in outputs if not unicodedata.combining(__a						)]						)
													if self.do_lower_case:
																				__SCREAMING_SNAKE_CASE										=   outputs.lower()

													return outputs






						def 			_A      (       self ,				_A						):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   self.preprocess_text(__a						)
													__SCREAMING_SNAKE_CASE										=   self.sp_model.encode(__a ,				out_type=__a						)
													__SCREAMING_SNAKE_CASE										=   []
													for piece in pieces:
																				if len(__a						) > 1 and piece[-1] == str(','						) and piece[-2].isdigit():
																											__SCREAMING_SNAKE_CASE										=   self.sp_model.EncodeAsPieces(piece[:-1].replace(__a ,				''						)						)
																											if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																		if len(cur_pieces[0]						) == 1:
																																									__SCREAMING_SNAKE_CASE										=   cur_pieces[1:]
																																		else:
																																									__SCREAMING_SNAKE_CASE										=   cur_pieces[0][1:]
																											cur_pieces.append(piece[-1]						)
																											new_pieces.extend(__a						)
																				else:
																											new_pieces.append(__a						)

													return new_pieces






						def 			_A      (       self ,				_A						):







													'''simple docstring'''

													return self.sp_model.PieceToId(__a						)






						def 			_A      (       self ,				_A						):







													'''simple docstring'''

													return self.sp_model.IdToPiece(__a						)






						def 			_A      (       self ,				_A						):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   ''.join(__a						).replace(__a ,				' '						).strip()
													return out_string






						def 			_A      (       self ,				_A ,				_A = False ,				_A = None ,				_A = True ,				**_A ,				):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   kwargs.pop('use_source_tokenizer' ,				__a						)

													__SCREAMING_SNAKE_CASE										=   self.convert_ids_to_tokens(__a ,				skip_special_tokens=__a						)

													# To avoid mixing byte-level and unicode for byte-level BPT
													# we need to build string separately for added tokens and byte-level tokens
													# cf. https://github.com/huggingface/transformers/issues/1133
													__SCREAMING_SNAKE_CASE										=   []
													__SCREAMING_SNAKE_CASE										=   []
													for token in filtered_tokens:
																				if skip_special_tokens and token in self.all_special_ids:
																											continue
																				if token in self.added_tokens_encoder:
																											if current_sub_text:
																																		sub_texts.append(self.convert_tokens_to_string(__a						)						)
																																		__SCREAMING_SNAKE_CASE										=   []
																											sub_texts.append(__a						)
																				else:
																											current_sub_text.append(__a						)
													if current_sub_text:
																				sub_texts.append(self.convert_tokens_to_string(__a						)						)

													# Mimic the behavior of the Rust tokenizer:
													# By default, there are no spaces between special tokens
													__SCREAMING_SNAKE_CASE										=   ''.join(__a						)

													__SCREAMING_SNAKE_CASE										=   (
													    clean_up_tokenization_spaces
													    if clean_up_tokenization_spaces is not None
													    else self.clean_up_tokenization_spaces
													)
													if clean_up_tokenization_spaces:
																				__SCREAMING_SNAKE_CASE										=   self.clean_up_tokenization(__a						)
																				return clean_text
													else:
																				return text






						def 			_A      (       self ,				_A ,				_A = None						):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
													__SCREAMING_SNAKE_CASE										=   [self.cls_token_id]
													if token_ids_a is None:
																				return token_ids_a + sep + cls
													return token_ids_a + sep + token_ids_a + sep + cls






						def 			_A      (       self ,				_A ,				_A = None ,				_A = False						):







													'''simple docstring'''

													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=__a ,				token_ids_a=__a ,				already_has_special_tokens=__a						)

													if token_ids_a is not None:
																				return ([0] * len(__a						)) + [1] + ([0] * len(__a						)) + [1, 1]
													return ([0] * len(__a						)) + [1, 1]






						def 			_A      (       self ,				_A ,				_A = None						):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
													__SCREAMING_SNAKE_CASE										=   [2]

													if token_ids_a is None:
																				return len(token_ids_a + sep						) * [0] + cls_segment_id
													return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id






						def 			_A      (       self ,				_A ,				_A = None						):







													'''simple docstring'''

													if not os.path.isdir(__a						):
																				logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
																				return
													__SCREAMING_SNAKE_CASE										=   os.path.join(
													    __a ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

													if os.path.abspath(self.vocab_file						) != os.path.abspath(__a						) and os.path.isfile(self.vocab_file						):
																				copyfile(self.vocab_file ,				__a						)
													elif not os.path.isfile(self.vocab_file						):
																				with open(__a ,				'wb'						) as fi:
																											__SCREAMING_SNAKE_CASE										=   self.sp_model.serialized_model_proto()
																											fi.write(__a						)

													return (out_vocab_file,)










"""simple docstring"""


import heapq as hq
import math
from collections.abc import Iterator
class     _UpperCamelCase    :
							'''simple docstring'''







							def __init__(	self       ,  __a      ):
										__lowerCAmelCase							       =     str(id_      )
										__lowerCAmelCase							       =     None
										__lowerCAmelCase							       =     None
										__lowerCAmelCase							       =     []
										__lowerCAmelCase							       =     {}  # {vertex:distance}

							def __lt__(	self       ,  __a      ):
										return self.key < other.key

							def __repr__(	self      ):
										return self.id

							def      snake_case							(	self       ,  __a      ):
										self.neighbors.append(__a      )



							def      snake_case							(	self       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     weight


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase       ,					_UpperCamelCase       ,					_UpperCamelCase ):

			'''simple docstring'''




			graph[a - 1].add_neighbor(graph[b - 1] )
			graph[b - 1].add_neighbor(graph[a - 1] )
			# add the edges:
			graph[a - 1].add_edge(graph[b - 1]       ,					_UpperCamelCase )
			graph[b - 1].add_edge(graph[a - 1]       ,					_UpperCamelCase )


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase ):

			'''simple docstring'''




			__lowerCAmelCase							       =     []
			for u in graph:
						__lowerCAmelCase							       =     math.inf
						__lowerCAmelCase							       =     None
			__lowerCAmelCase							       =     0
			__lowerCAmelCase							       =     graph[:]
			while q:
						__lowerCAmelCase							       =     min(_UpperCamelCase )
						q.remove(_UpperCamelCase )
						for v in u.neighbors:
									if (v in q) and (u.edges[v.id] < v.key):
												__lowerCAmelCase							       =     u
												__lowerCAmelCase							       =     u.edges[v.id]
			for i in range(1       ,					len(_UpperCamelCase ) ):
						a.append((int(graph[i].id ) + 1, int(graph[i].pi.id ) + 1) )
			return a


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase ):

			'''simple docstring'''




			for u in graph:
						__lowerCAmelCase							       =     math.inf
						__lowerCAmelCase							       =     None
			__lowerCAmelCase							       =     0

			__lowerCAmelCase							       =     list(_UpperCamelCase )
			hq.heapify(_UpperCamelCase )

			while h:
						__lowerCAmelCase							       =     hq.heappop(_UpperCamelCase )
						for v in u.neighbors:
									if (v in h) and (u.edges[v.id] < v.key):
												__lowerCAmelCase							       =     u
												__lowerCAmelCase							       =     u.edges[v.id]
												hq.heapify(_UpperCamelCase )

			for i in range(1       ,					len(_UpperCamelCase ) ):
						yield (int(graph[i].id ) + 1, int(graph[i].pi.id ) + 1)


def _lowerCamelCase      (						):

			'''simple docstring'''






if __name__ == "__main__":
	import doctest

	doctest.testmod()




