





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
					import torch


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


UpperCamelCase__           =			logging.get_logger(__name__)
UpperCamelCase__           =			{"vocab_file": "spiece.model"}

UpperCamelCase__           =			{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

UpperCamelCase__           =			{
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Tuple      =							VOCAB_FILES_NAMES
							snake_case  :   Any      =							PRETRAINED_VOCAB_FILES_MAP
							snake_case  :   Union[str, Any]      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case  :   str      =							["""input_ids""", """attention_mask"""]






							def __init__(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=False	,			__lowerCAmelCase=False	,			__lowerCAmelCase=None	,			__lowerCAmelCase=None	,			__lowerCAmelCase=None	,			__lowerCAmelCase=None	,			__lowerCAmelCase = None	,			**__lowerCAmelCase	,			):
														UpperCamelCase__					  =      {} if sp_model_kwargs is None else sp_model_kwargs

														UpperCamelCase__					  =      kwargs.get("""name_or_path"""	)
														if name_or_path is None:
																					logger.warning(
																					    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
																					    """ you are testing the model, this can safely be ignored"""	)
																					UpperCamelCase__					  =      """None"""

														# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
														UpperCamelCase__					  =      """<|endoftext|>""" if eos_token is None else eos_token
														UpperCamelCase__					  =      """<unk>""" if unk_token is None else unk_token
														if "gpt-sw3-7b" in name_or_path:
																					UpperCamelCase__					  =      unk_token if pad_token is None else pad_token
																					UpperCamelCase__					  =      eos_token if bos_token is None else bos_token
														else:
																					UpperCamelCase__					  =      """<pad>""" if pad_token is None else pad_token
																					UpperCamelCase__					  =      """<s>""" if bos_token is None else bos_token

														super().__init__(
														    do_lower_case=__lowerCAmelCase	,			remove_space=__lowerCAmelCase	,			keep_accents=__lowerCAmelCase	,			bos_token=__lowerCAmelCase	,			eos_token=__lowerCAmelCase	,			unk_token=__lowerCAmelCase	,			pad_token=__lowerCAmelCase	,			sp_model_kwargs=self.sp_model_kwargs	,			**__lowerCAmelCase	,			)

														UpperCamelCase__					  =      do_lower_case
														UpperCamelCase__					  =      remove_space
														UpperCamelCase__					  =      keep_accents
														UpperCamelCase__					  =      vocab_file

														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(__lowerCAmelCase	)

														# Used for whitespace normalization in input texts
														# fmt : off
														UpperCamelCase__					  =      {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
														# fmt : on

														# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
														UpperCamelCase__					  =      re.compile(
														    f"""[{"".join(map(__lowerCAmelCase	,			list(range(0	,			9	)	) + list(range(11	,			32	)	) + list(range(127	,			160	)	) + [160, 173, 8203]	)	)}]"""	)






							def __getstate__(    self	):
														UpperCamelCase__					  =      self.__dict__.copy()
														UpperCamelCase__					  =      None
														return state






							def __setstate__(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      d

														# for backward compatibility
														if not hasattr(self	,			"""sp_model_kwargs"""	):
																					UpperCamelCase__					  =      {}

														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(self.vocab_file	)






							@property
							# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
							def 					_lowerCamelCase		(    self	):
														return len(self.sp_model	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      self.non_printing_characters_re.sub(""""""	,			__lowerCAmelCase	)

														# Normalize whitespaces
														UpperCamelCase__					  =      """""".join([char if char not in self.whitespaces else """ """ for char in text]	)

														# NFC Unicode normalization
														UpperCamelCase__					  =      unicodedata.normalize("""NFC"""	,			__lowerCAmelCase	)
														return text






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      self.preprocess_text(__lowerCAmelCase	)
														return self.sp_model.encode(__lowerCAmelCase	,			out_type=__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.sp_model.PieceToId(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.sp_model.IdToPiece(__lowerCAmelCase	)






							@staticmethod
							def 					_lowerCamelCase		(    __lowerCAmelCase	):
														return out_string






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      """"""
														UpperCamelCase__					  =      False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																												if not prev_is_special:
																																			out_string += " "

																												out_string += self.sp_model.decode(__lowerCAmelCase	) + token
																												UpperCamelCase__					  =      True
																												UpperCamelCase__					  =      []
																					else:
																												current_sub_tokens.append(__lowerCAmelCase	)
																												UpperCamelCase__					  =      False
														out_string += self.sp_model.decode(__lowerCAmelCase	)

														return out_string






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      {self.convert_ids_to_tokens(__lowerCAmelCase	): i for i in range(self.vocab_size	)}
														vocab.update(self.added_tokens_encoder	)
														return vocab






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														if not os.path.isdir(__lowerCAmelCase	):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
																					return
														UpperCamelCase__					  =      os.path.join(
														    __lowerCAmelCase	,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

														if os.path.abspath(self.vocab_file	) != os.path.abspath(__lowerCAmelCase	) and os.path.isfile(self.vocab_file	):
																					copyfile(self.vocab_file	,			__lowerCAmelCase	)
														elif not os.path.isfile(self.vocab_file	):
																					with open(__lowerCAmelCase	,			"""wb"""	) as fi:
																												UpperCamelCase__					  =      self.sp_model.serialized_model_proto()
																												fi.write(__lowerCAmelCase	)

														return (out_vocab_file,)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = False	):

														if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	):
																					UpperCamelCase__					  =      self.preprocess_text(__lowerCAmelCase	)
																					UpperCamelCase__					  =      self.sp_model.encode(__lowerCAmelCase	)
														else:
																					UpperCamelCase__					  =      [self.preprocess_text(__lowerCAmelCase	) for t in text]
																					UpperCamelCase__					  =      self.sp_model.encode(__lowerCAmelCase	)

														if return_tensors is True or return_tensors == "pt":
																					UpperCamelCase__					  =      torch.tensor(__lowerCAmelCase	)

														return token_ids






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.sp_model.decode(__lowerCAmelCase	)







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      [f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
														UpperCamelCase__					  =      (
														    f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(__lowerCAmelCase	) + f"""{self.bos_token}Bot:"""
														)
														return self.encode(text=__lowerCAmelCase	)











import copy
import random

from transformers import CLIPTokenizer

class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def __init__(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														super().__init__(*__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      {}






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      super().add_tokens(__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	)
														if num_added_tokens == 0:
																					raise ValueError(
																					    f"""The tokenizer already contains the token {placeholder_token}. Please pass a different"""
																					    """ `placeholder_token` that is not already in the tokenizer."""	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			__lowerCAmelCase=1	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      []
														if num_vec_per_token == 1:
																					self.try_adding_tokens(__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	)
																					output.append(__lowerCAmelCase	)
														else:
																					UpperCamelCase__					  =      []
																					for i in range(__lowerCAmelCase	):
																												UpperCamelCase__					  =      placeholder_token + f"""_{i}"""
																												self.try_adding_tokens(__lowerCAmelCase	,			*__lowerCAmelCase	,			**__lowerCAmelCase	)
																												output.append(__lowerCAmelCase	)
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
														for token in self.token_map:
																					if token in placeholder_token:
																												raise ValueError(
																												    f"""The tokenizer already has placeholder token {token} that can get confused with"""
																												    f""" {placeholder_token}keep placeholder tokens independent"""	)
														UpperCamelCase__					  =      output






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=1.0	):
														if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	):
																					UpperCamelCase__					  =      []
																					for i in range(len(__lowerCAmelCase	)	):
																												output.append(self.replace_placeholder_tokens_in_text(text[i]	,			vector_shuffle=__lowerCAmelCase	)	)
																					return output
														for placeholder_token in self.token_map:
																					if placeholder_token in text:
																												UpperCamelCase__					  =      self.token_map[placeholder_token]
																												UpperCamelCase__					  =      tokens[: 1 + int(len(__lowerCAmelCase	) * prop_tokens_to_load	)]
																												if vector_shuffle:
																																			UpperCamelCase__					  =      copy.copy(__lowerCAmelCase	)
																																			random.shuffle(__lowerCAmelCase	)
																												UpperCamelCase__					  =      text.replace(__lowerCAmelCase	,			""" """.join(__lowerCAmelCase	)	)
														return text






							def __call__(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=1.0	,			**__lowerCAmelCase	):
														return super().__call__(
														    self.replace_placeholder_tokens_in_text(
														        __lowerCAmelCase	,			vector_shuffle=__lowerCAmelCase	,			prop_tokens_to_load=__lowerCAmelCase	)	,			*__lowerCAmelCase	,			**__lowerCAmelCase	,			)







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=1.0	,			**__lowerCAmelCase	):
														return super().encode(
														    self.replace_placeholder_tokens_in_text(
														        __lowerCAmelCase	,			vector_shuffle=__lowerCAmelCase	,			prop_tokens_to_load=__lowerCAmelCase	)	,			*__lowerCAmelCase	,			**__lowerCAmelCase	,			)




