


"""simple docstring"""






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
							import torch


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


UpperCAmelCase    :     Any   						=		logging.get_logger(__name__)
UpperCAmelCase    :     str   						=		{'vocab_file': 'spiece.model'}

UpperCAmelCase    :     Tuple   						=		{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

UpperCAmelCase    :     Dict   						=		{
    'AI-Sweden/gpt-sw3-126m': 2048,
    'AI-Sweden/gpt-sw3-350m': 2048,
    'AI-Sweden/gpt-sw3-1.6b': 2048,
    'AI-Sweden/gpt-sw3-6.7b': 2048,
    'AI-Sweden/gpt-sw3-20b': 2048,
}





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					__a	       =							VOCAB_FILES_NAMES
					__a	       =							PRETRAINED_VOCAB_FILES_MAP
					__a	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__a	       =							["""input_ids""", """attention_mask"""]






					def __init__(       self    : Any ,       UpperCamelCase    : Dict ,       UpperCamelCase    : Any=False ,       UpperCamelCase    : int=False ,       UpperCamelCase    : Tuple=False ,       UpperCamelCase    : List[str]=None ,       UpperCamelCase    : Any=None ,       UpperCamelCase    : List[str]=None ,       UpperCamelCase    : Optional[Any]=None ,       UpperCamelCase    : Optional[Dict[str, Any]] = None ,       **UpperCamelCase    : Dict ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					{} if sp_model_kwargs is None else sp_model_kwargs

												__UpperCAmelCase   :						int      							=					kwargs.get("""name_or_path"""  )
												if name_or_path is None:
																			logger.warning(
																			    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
																			    """ you are testing the model, this can safely be ignored"""  )
																			__UpperCAmelCase   :						str      							=					"""None"""

												# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
												__UpperCAmelCase   :						List[Any]      							=					"""<|endoftext|>""" if eos_token is None else eos_token
												__UpperCAmelCase   :						List[Any]      							=					"""<unk>""" if unk_token is None else unk_token
												if "gpt-sw3-7b" in name_or_path:
																			__UpperCAmelCase   :						List[Any]      							=					unk_token if pad_token is None else pad_token
																			__UpperCAmelCase   :						Dict      							=					eos_token if bos_token is None else bos_token
												else:
																			__UpperCAmelCase   :						Any      							=					"""<pad>""" if pad_token is None else pad_token
																			__UpperCAmelCase   :						List[str]      							=					"""<s>""" if bos_token is None else bos_token

												super().__init__(
												    do_lower_case=UpperCamelCase ,       remove_space=UpperCamelCase ,       keep_accents=UpperCamelCase ,       bos_token=UpperCamelCase ,       eos_token=UpperCamelCase ,       unk_token=UpperCamelCase ,       pad_token=UpperCamelCase ,       sp_model_kwargs=self.sp_model_kwargs ,       **UpperCamelCase ,       )

												__UpperCAmelCase   :						Optional[int]      							=					do_lower_case
												__UpperCAmelCase   :						int      							=					remove_space
												__UpperCAmelCase   :						Optional[int]      							=					keep_accents
												__UpperCAmelCase   :						List[Any]      							=					vocab_file

												__UpperCAmelCase   :						int      							=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
												self.sp_model.Load(UpperCamelCase  )

												# Used for whitespace normalization in input texts
												# fmt : off
												__UpperCAmelCase   :						str      							=					{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
												# fmt : on

												# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
												__UpperCAmelCase   :						Optional[int]      							=					re.compile(
												    f'''[{"".join(map(UpperCamelCase ,       list(range(0 ,       9  )  ) + list(range(11 ,       32  )  ) + list(range(127 ,       160  )  ) + [160, 173, 8_203]  )  )}]'''  )






					def __getstate__(       self    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					self.__dict__.copy()
												__UpperCAmelCase   :						Union[str, Any]      							=					None
												return state






					def __setstate__(       self    : List[str] ,       UpperCamelCase    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					d

												# for backward compatibility
												if not hasattr(self ,       """sp_model_kwargs"""  ):
																			__UpperCAmelCase   :						Any      							=					{}

												__UpperCAmelCase   :						Optional[Any]      							=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
												self.sp_model.Load(self.vocab_file  )






					@property
					# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												return len(self.sp_model  )






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					self.non_printing_characters_re.sub("""""" ,       UpperCamelCase  )

												# Normalize whitespaces
												__UpperCAmelCase   :						Tuple      							=					"""""".join([char if char not in self.whitespaces else """ """ for char in text]  )

												# NFC Unicode normalization
												__UpperCAmelCase   :						Any      							=					unicodedata.normalize("""NFC""" ,       UpperCamelCase  )
												return text






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : str ,       **UpperCamelCase    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					self.preprocess_text(UpperCamelCase  )
												return self.sp_model.encode(UpperCamelCase ,       out_type=UpperCamelCase  )






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												return self.sp_model.PieceToId(UpperCamelCase  )






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												return self.sp_model.IdToPiece(UpperCamelCase  )






					@staticmethod
					def lowerCamelCase__							(       UpperCamelCase    : str  ):
												'''simple docstring'''






												return out_string






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : List[str]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					[]
												__UpperCAmelCase   :						List[Any]      							=					""""""
												__UpperCAmelCase   :						List[Any]      							=					False
												for token in tokens:
																			# make sure that special tokens are not decoded using sentencepiece model
																			if token in self.all_special_tokens:
																										# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																										if not prev_is_special:
																																	out_string += " "

																										out_string += self.sp_model.decode(UpperCamelCase  ) + token
																										__UpperCAmelCase   :						List[str]      							=					True
																										__UpperCAmelCase   :						Optional[Any]      							=					[]
																			else:
																										current_sub_tokens.append(UpperCamelCase  )
																										__UpperCAmelCase   :						Optional[Any]      							=					False
												out_string += self.sp_model.decode(UpperCamelCase  )

												return out_string






					def lowerCamelCase__							(       self    : Tuple  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					{self.convert_ids_to_tokens(UpperCamelCase  ): i for i in range(self.vocab_size  )}
												vocab.update(self.added_tokens_encoder  )
												return vocab






					def lowerCamelCase__							(       self    : Tuple ,       UpperCamelCase    : str ,       UpperCamelCase    : Optional[str] = None  ):
												'''simple docstring'''






												if not os.path.isdir(UpperCamelCase  ):
																			logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
																			return
												__UpperCAmelCase   :						Dict      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

												if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
																			copyfile(self.vocab_file ,       UpperCamelCase  )
												elif not os.path.isfile(self.vocab_file  ):
																			with open(UpperCamelCase ,       """wb"""  ) as fi:
																										__UpperCAmelCase   :						Any      							=					self.sp_model.serialized_model_proto()
																										fi.write(UpperCamelCase  )

												return (out_vocab_file,)






					def lowerCamelCase__							(       self    : str ,       UpperCamelCase    : Union[str, List[str]] ,       UpperCamelCase    : Union[str, bool] = False  ):
												'''simple docstring'''






												if isinstance(UpperCamelCase ,       UpperCamelCase  ):
																			__UpperCAmelCase   :						Union[str, Any]      							=					self.preprocess_text(UpperCamelCase  )
																			__UpperCAmelCase   :						Any      							=					self.sp_model.encode(UpperCamelCase  )
												else:
																			__UpperCAmelCase   :						Dict      							=					[self.preprocess_text(UpperCamelCase  ) for t in text]
																			__UpperCAmelCase   :						Any      							=					self.sp_model.encode(UpperCamelCase  )

												if return_tensors is True or return_tensors == "pt":
																			__UpperCAmelCase   :						Union[str, Any]      							=					torch.tensor(UpperCamelCase  )

												return token_ids






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : Union[int, List[int]]  ):
												'''simple docstring'''






												return self.sp_model.decode(UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[int] ,       UpperCamelCase    : "Conversation"  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					[f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
												__UpperCAmelCase   :						Dict      							=					(
												    f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(UpperCamelCase  ) + f'''{self.bos_token}Bot:'''
												)
												return self.encode(text=UpperCamelCase  )




"""simple docstring"""






import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
							import ctypes
							import msvcrt  # noqa





							class      lowerCamelCase__   (     ctypes.Structure				):







												"""simple docstring"""


												__a	       =							[("""size""", ctypes.c_int), ("""visible""", ctypes.c_byte)]
def      lowerCamelCase       (     )		->       Optional[int]:

							'''simple docstring'''



							if os.name == "nt":
														__UpperCAmelCase   :						Dict      							=					CursorInfo()
														__UpperCAmelCase   :						Any      							=					ctypes.windll.kernelaa.GetStdHandle(-1_1							)
														ctypes.windll.kernelaa.GetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
														__UpperCAmelCase   :						Tuple      							=					False
														ctypes.windll.kernelaa.SetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
							elif os.name == "posix":
														sys.stdout.write("""\033[?25l"""							)
														sys.stdout.flush()
def      lowerCamelCase       (     )		->       Optional[int]:

							'''simple docstring'''



							if os.name == "nt":
														__UpperCAmelCase   :						str      							=					CursorInfo()
														__UpperCAmelCase   :						int      							=					ctypes.windll.kernelaa.GetStdHandle(-1_1							)
														ctypes.windll.kernelaa.GetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
														__UpperCAmelCase   :						Union[str, Any]      							=					True
														ctypes.windll.kernelaa.SetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
							elif os.name == "posix":
														sys.stdout.write("""\033[?25h"""							)
														sys.stdout.flush()
@contextmanager
def      lowerCamelCase       (     )		->       str:

							'''simple docstring'''



							try:
														hide_cursor()
														yield
							finally:
														show_cursor()
