


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase						=		logging.get_logger(__name__)
lowerCamelCase						=		{"vocab_file": "spiece.model"}

lowerCamelCase						=		{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

lowerCamelCase						=		{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

lowerCamelCase						=		"▁"





class 					_a							(				a_):
	_a  :  Tuple          =							VOCAB_FILES_NAMES
	_a  :  Optional[int]          =							PRETRAINED_VOCAB_FILES_MAP
	_a  :  Tuple          =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




	def __init__(					self						:		Any		,					_SCREAMING_SNAKE_CASE						:		Optional[int]		,					_SCREAMING_SNAKE_CASE						:		int=True		,					_SCREAMING_SNAKE_CASE						:		Optional[Any]=True		,					_SCREAMING_SNAKE_CASE						:		Tuple=False		,					_SCREAMING_SNAKE_CASE						:		List[Any]="[CLS]"		,					_SCREAMING_SNAKE_CASE						:		List[Any]="[SEP]"		,					_SCREAMING_SNAKE_CASE						:		Optional[Any]="<unk>"		,					_SCREAMING_SNAKE_CASE						:		List[str]="[SEP]"		,					_SCREAMING_SNAKE_CASE						:		str="<pad>"		,					_SCREAMING_SNAKE_CASE						:		Optional[int]="[CLS]"		,					_SCREAMING_SNAKE_CASE						:		List[Any]="[MASK]"		,					_SCREAMING_SNAKE_CASE						:		Union[str, Any] = None		,					**_SCREAMING_SNAKE_CASE						:		Optional[int]		,					)->       List[str]:
			lowerCAmelCase__   :			Dict         =					(
			    AddedToken(_A		,					lstrip=_A		,					rstrip=_A		,					normalized=_A		)
			    if isinstance(_A		,					_A		)
			    else mask_token
			)

			lowerCAmelCase__   :			Union[str, Any]         =					{} if sp_model_kwargs is None else sp_model_kwargs

			super().__init__(
			    do_lower_case=_A		,					remove_space=_A		,					keep_accents=_A		,					bos_token=_A		,					eos_token=_A		,					unk_token=_A		,					sep_token=_A		,					pad_token=_A		,					cls_token=_A		,					mask_token=_A		,					sp_model_kwargs=self.sp_model_kwargs		,					**_A		,					)

			lowerCAmelCase__   :			Optional[int]         =					do_lower_case
			lowerCAmelCase__   :			Optional[int]         =					remove_space
			lowerCAmelCase__   :			str         =					keep_accents
			lowerCAmelCase__   :			Dict         =					vocab_file

			lowerCAmelCase__   :			List[Any]         =					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
			self.sp_model.Load(_A		)




	@property
	def 							UpperCAmelCase__(					self						:		Union[str, Any]		)->       Optional[Any]:
			return len(self.sp_model		)




	def 							UpperCAmelCase__(					self						:		Any		)->       Optional[int]:
			lowerCAmelCase__   :			Union[str, Any]         =					{self.convert_ids_to_tokens(_A		): i for i in range(self.vocab_size		)}
			vocab.update(self.added_tokens_encoder		)
			return vocab




	def __getstate__(					self						:		Optional[int]		)->       Any:
			lowerCAmelCase__   :			str         =					self.__dict__.copy()
			lowerCAmelCase__   :			Optional[int]         =					None
			return state




	def __setstate__(					self						:		Any		,					_SCREAMING_SNAKE_CASE						:		List[Any]		)->       Tuple:
			lowerCAmelCase__   :			Any         =					d

			# for backward compatibility
			if not hasattr(self		,					'''sp_model_kwargs'''		):
					lowerCAmelCase__   :			List[str]         =					{}

			lowerCAmelCase__   :			Optional[Any]         =					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
			self.sp_model.Load(self.vocab_file		)




	def 							UpperCAmelCase__(					self						:		int		,					_SCREAMING_SNAKE_CASE						:		Union[str, Any]		)->       int:







			if self.remove_space:
					lowerCAmelCase__   :			int         =					''' '''.join(inputs.strip().split()		)
			else:
					lowerCAmelCase__   :			Optional[Any]         =					inputs
			lowerCAmelCase__   :			Union[str, Any]         =					outputs.replace('''``'''		,					'''"'''		).replace('''\'\''''		,					'''"'''		)

			if not self.keep_accents:
					lowerCAmelCase__   :			Dict         =					unicodedata.normalize('''NFKD'''		,					_A		)
					lowerCAmelCase__   :			str         =					''''''.join([c for c in outputs if not unicodedata.combining(_A		)]		)
			if self.do_lower_case:
					lowerCAmelCase__   :			Optional[int]         =					outputs.lower()

			return outputs




	def 							UpperCAmelCase__(					self						:		Tuple		,					_SCREAMING_SNAKE_CASE						:		Union[str, Any]		)->       Any:
			lowerCAmelCase__   :			List[str]         =					self.preprocess_text(_A		)
			lowerCAmelCase__   :			Dict         =					self.sp_model.encode(_A		,					out_type=_A		)
			lowerCAmelCase__   :			Dict         =					[]
			for piece in pieces:
					if len(_A		) > 1 and piece[-1] == str(''','''		) and piece[-2].isdigit():
							lowerCAmelCase__   :			Tuple         =					self.sp_model.EncodeAsPieces(piece[:-1].replace(_A		,					''''''		)		)
							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
									if len(cur_pieces[0]		) == 1:
											lowerCAmelCase__   :			int         =					cur_pieces[1:]
									else:
											lowerCAmelCase__   :			int         =					cur_pieces[0][1:]
							cur_pieces.append(piece[-1]		)
							new_pieces.extend(_A		)
					else:
							new_pieces.append(_A		)

			return new_pieces




	def 							UpperCAmelCase__(					self						:		Union[str, Any]		,					_SCREAMING_SNAKE_CASE						:		Optional[Any]		)->       Tuple:
			return self.sp_model.PieceToId(_A		)




	def 							UpperCAmelCase__(					self						:		str		,					_SCREAMING_SNAKE_CASE						:		Dict		)->       str:
			return self.sp_model.IdToPiece(_A		)




	def 							UpperCAmelCase__(					self						:		List[Any]		,					_SCREAMING_SNAKE_CASE						:		List[str]		)->       Optional[Any]:
			lowerCAmelCase__   :			List[str]         =					[]
			lowerCAmelCase__   :			Any         =					''''''
			lowerCAmelCase__   :			Optional[Any]         =					False
			for token in tokens:
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
							if not prev_is_special:
									out_string += " "
							out_string += self.sp_model.decode(_A		) + token
							lowerCAmelCase__   :			Any         =					True
							lowerCAmelCase__   :			List[str]         =					[]
					else:
							current_sub_tokens.append(_A		)
							lowerCAmelCase__   :			Union[str, Any]         =					False
			out_string += self.sp_model.decode(_A		)
			return out_string.strip()




	def 							UpperCAmelCase__(					self						:		Optional[Any]		,					_SCREAMING_SNAKE_CASE						:		Any		,					_SCREAMING_SNAKE_CASE						:		Union[str, Any] = None		)->       Any:
			lowerCAmelCase__   :			Optional[Any]         =					[self.sep_token_id]
			lowerCAmelCase__   :			Optional[int]         =					[self.cls_token_id]
			if token_ids_a is None:
					return cls + token_ids_a + sep
			return cls + token_ids_a + sep + token_ids_a + sep




	def 							UpperCAmelCase__(					self						:		List[str]		,					_SCREAMING_SNAKE_CASE						:		Dict		,					_SCREAMING_SNAKE_CASE						:		Dict = None		,					_SCREAMING_SNAKE_CASE						:		Optional[int] = False		)->       str:







			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=_A		,					token_ids_a=_A		,					already_has_special_tokens=_A		)

			if token_ids_a is not None:
					return [1] + ([0] * len(_A		)) + [1] + ([0] * len(_A		)) + [1]
			return [1] + ([0] * len(_A		)) + [1]




	def 							UpperCAmelCase__(					self						:		List[Any]		,					_SCREAMING_SNAKE_CASE						:		Any		,					_SCREAMING_SNAKE_CASE						:		Dict = None		)->       Dict:
			lowerCAmelCase__   :			Any         =					[self.sep_token_id]
			lowerCAmelCase__   :			Optional[int]         =					[self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep		) * [0]
			return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




	def 							UpperCAmelCase__(					self						:		str		,					_SCREAMING_SNAKE_CASE						:		Tuple		,					_SCREAMING_SNAKE_CASE						:		Optional[int] = None		)->       List[Any]:







			if not os.path.isdir(_A		):
					logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
					return
			lowerCAmelCase__   :			List[Any]         =					os.path.join(
			    _A		,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

			if os.path.abspath(self.vocab_file		) != os.path.abspath(_A		) and os.path.isfile(self.vocab_file		):
					copyfile(self.vocab_file		,					_A		)
			elif not os.path.isfile(self.vocab_file		):
					with open(_A		,					'''wb'''		) as fi:
							lowerCAmelCase__   :			Dict         =					self.sp_model.serialized_model_proto()
							fi.write(_A		)

			return (out_vocab_file,)











import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


__A				:    Dict       =   logging.get_logger(__name__)

__A				:    Any       =   {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A				:    Tuple       =   {
    "vocab_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"
        ),
        "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt",
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli": (
            "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"
        ),
    },
}

__A				:    List[Any]       =   {
    "squeezebert/squeezebert-uncased": 512,
    "squeezebert/squeezebert-mnli": 512,
    "squeezebert/squeezebert-mnli-headless": 512,
}


__A				:    List[Any]       =   {
    "squeezebert/squeezebert-uncased": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli-headless": {"do_lower_case": True},
}

class 	A_       (a_			):
	UpperCAmelCase__					=			VOCAB_FILES_NAMES
	UpperCAmelCase__					=			PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase__					=			PRETRAINED_INIT_CONFIGURATION
	UpperCAmelCase__					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCAmelCase__					=			SqueezeBertTokenizer
	def __init__(    self							,					_A=None							,					_A=None							,					_A=True							,					_A="[UNK]"							,					_A="[SEP]"							,					_A="[PAD]"							,					_A="[CLS]"							,					_A="[MASK]"							,					_A=True							,					_A=None							,					**_A							,					):






				'''simple docstring'''







				super().__init__(
				    _A							,					tokenizer_file=_A							,					do_lower_case=_A							,					unk_token=_A							,					sep_token=_A							,					pad_token=_A							,					cls_token=_A							,					mask_token=_A							,					tokenize_chinese_chars=_A							,					strip_accents=_A							,					**_A							,					)

				UpperCAmelCase   		=				json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
				if (
				    normalizer_state.get('''lowercase'''							,					_A		) != do_lower_case
				    or normalizer_state.get('''strip_accents'''							,					_A		) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''							,					_A		) != tokenize_chinese_chars
				):
							UpperCAmelCase   		=				getattr(_A							,					normalizer_state.pop('''type'''		)		)
							UpperCAmelCase   		=				do_lower_case
							UpperCAmelCase   		=				strip_accents
							UpperCAmelCase   		=				tokenize_chinese_chars
							UpperCAmelCase   		=				normalizer_class(**_A		)

				UpperCAmelCase   		=				do_lower_case
	def      _lowercase				(    self							,					_A							,					_A=None		):






				'''simple docstring'''







				UpperCAmelCase   		=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output
	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				[self.sep_token_id]
				UpperCAmelCase   		=				[self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				self._tokenizer.model.save(_A							,					name=_A		)
				return tuple(_A		)



