





'''simple docstring'''

def 					a__				(				lowercase					:     Any,					lowercase					:     str      )		->		Optional[Any]:





     """simple docstring"""

     if density <= 0:
          raise ValueError('''Impossible fluid density'''      )
     if bulk_modulus <= 0:
          raise ValueError('''Impossible bulk modulus'''      )

     return (bulk_modulus / density) ** 0.5


if __name__ == "__main__":
   import doctest

   doctest.testmod()






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_UpperCamelCase 			=		logging.get_logger(__name__)
_UpperCamelCase 			=		{"vocab_file": "spiece.model"}

_UpperCamelCase 			=		{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_UpperCamelCase 			=		{
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}







class   __lowercase      (_UpperCAmelCase ):
 _UpperCamelCase			  =  VOCAB_FILES_NAMES
 _UpperCamelCase			  =  PRETRAINED_VOCAB_FILES_MAP
 _UpperCamelCase			  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _UpperCamelCase			  =  ["""input_ids""", """attention_mask"""]
 def __init__(							self      ,							A_      ,							A_=False      ,							A_=False      ,							A_=False      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_ = None      ,							**A_      ,							)					->None:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					{} if sp_model_kwargs is None else sp_model_kwargs

       __lowerCAmelCase					:						int					=					kwargs.get('''name_or_path'''					)
       if name_or_path is None:
             logger.warning(
                 '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                 ''' you are testing the model, this can safely be ignored'''					)
             __lowerCAmelCase					:						Union[str, Any]					=					'''None'''

       # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
       __lowerCAmelCase					:						str					=					'''<|endoftext|>''' if eos_token is None else eos_token
       __lowerCAmelCase					:						Any					=					'''<unk>''' if unk_token is None else unk_token
       if "gpt-sw3-7b" in name_or_path:
             __lowerCAmelCase					:						Dict					=					unk_token if pad_token is None else pad_token
             __lowerCAmelCase					:						int					=					eos_token if bos_token is None else bos_token
       else:
             __lowerCAmelCase					:						Optional[int]					=					'''<pad>''' if pad_token is None else pad_token
             __lowerCAmelCase					:						List[str]					=					'''<s>''' if bos_token is None else bos_token

       super().__init__(
           do_lower_case=A_      ,							remove_space=A_      ,							keep_accents=A_      ,							bos_token=A_      ,							eos_token=A_      ,							unk_token=A_      ,							pad_token=A_      ,							sp_model_kwargs=self.sp_model_kwargs      ,							**A_      ,							)

       __lowerCAmelCase					:						Union[str, Any]					=					do_lower_case
       __lowerCAmelCase					:						Union[str, Any]					=					remove_space
       __lowerCAmelCase					:						int					=					keep_accents
       __lowerCAmelCase					:						Union[str, Any]					=					vocab_file

       __lowerCAmelCase					:						int					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(A_					)

       # Used for whitespace normalization in input texts
       # fmt : off
       __lowerCAmelCase					:						List[Any]					=					{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
       # fmt : on

       # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
       __lowerCAmelCase					:						int					=					re.compile(
           f"""[{"".join(map(A_      ,							list(range(0      ,							9					)					) + list(range(11      ,							32					)					) + list(range(127      ,							160					)					) + [160, 173, 8203]					)					)}]"""					)
 def __getstate__(							self					)					->Dict:



       '''simple docstring'''





       __lowerCAmelCase					:						Union[str, Any]					=					self.__dict__.copy()
       __lowerCAmelCase					:						List[Any]					=					None
       return state
 def __setstate__(							self      ,							A_					)					->Tuple:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					d

       # for backward compatibility
       if not hasattr(self      ,							'''sp_model_kwargs'''					):
             __lowerCAmelCase					:						List[Any]					=					{}

       __lowerCAmelCase					:						List[str]					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(self.vocab_file					)
 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       return len(self.sp_model					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.non_printing_characters_re.sub(''''''      ,							A_					)

       # Normalize whitespaces
       __lowerCAmelCase					:						List[str]					=					''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]					)

       # NFC Unicode normalization
       __lowerCAmelCase					:						Tuple					=					unicodedata.normalize('''NFC'''      ,							A_					)
       return text
 def 	UpperCamelCase__  (							self      ,							A_      ,							**A_					)					->List[str]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.preprocess_text(A_					)
       return self.sp_model.encode(A_      ,							out_type=A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->int:



       '''simple docstring'''





       return self.sp_model.PieceToId(A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.IdToPiece(A_					)
 @staticmethod
 def 	UpperCamelCase__  (							A_					)					->str:



       '''simple docstring'''





       return out_string
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					[]
       __lowerCAmelCase					:						Tuple					=					''''''
       __lowerCAmelCase					:						int					=					False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                   if not prev_is_special:
                         out_string += " "

                   out_string += self.sp_model.decode(A_					) + token
                   __lowerCAmelCase					:						Optional[Any]					=					True
                   __lowerCAmelCase					:						Optional[int]					=					[]
             else:
                   current_sub_tokens.append(A_					)
                   __lowerCAmelCase					:						str					=					False
       out_string += self.sp_model.decode(A_					)

       return out_string
 def 	UpperCamelCase__  (							self					)					->Dict[str, int]:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					{self.convert_ids_to_tokens(A_					): i for i in range(self.vocab_size					)}
       vocab.update(self.added_tokens_encoder					)
       return vocab
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = None					)					->Tuple[str]:



       '''simple docstring'''





       if not os.path.isdir(A_					):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
             return
       __lowerCAmelCase					:						Any					=					os.path.join(
           A_      ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

       if os.path.abspath(self.vocab_file					) != os.path.abspath(A_					) and os.path.isfile(self.vocab_file					):
             copyfile(self.vocab_file      ,							A_					)
       elif not os.path.isfile(self.vocab_file					):
             with open(A_      ,							'''wb'''					) as fi:
                   __lowerCAmelCase					:						Dict					=					self.sp_model.serialized_model_proto()
                   fi.write(A_					)

       return (out_vocab_file,)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = False					)					->Union[List[int], List[List[int]], "torch.Tensor"]:



       '''simple docstring'''





       if isinstance(A_      ,							A_					):
             __lowerCAmelCase					:						Optional[Any]					=					self.preprocess_text(A_					)
             __lowerCAmelCase					:						Dict					=					self.sp_model.encode(A_					)
       else:
             __lowerCAmelCase					:						Dict					=					[self.preprocess_text(A_					) for t in text]
             __lowerCAmelCase					:						Optional[int]					=					self.sp_model.encode(A_					)

       if return_tensors is True or return_tensors == "pt":
             __lowerCAmelCase					:						Tuple					=					torch.tensor(A_					)

       return token_ids
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.decode(A_					)





 def 	UpperCamelCase__  (							self      ,							A_					)					->List[int]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					[f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
       __lowerCAmelCase					:						Any					=					(
           f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(A_					) + f"""{self.bos_token}Bot:"""
       )
       return self.encode(text=A_					)


