





'''simple docstring'''

from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import BaseOutput, is_torch_available, is_transformers_available
@dataclass
class 				__lowerCAmelCase     (    _UpperCAmelCase ):






       """simple docstring"""
       _snake_case	:     List[Any]										=    4_2
       _snake_case	:     Tuple										=    4_2


if is_transformers_available() and is_torch_available():
   from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_UpperCamelCase 			=		logging.get_logger(__name__)
_UpperCamelCase 			=		{"vocab_file": "spiece.model"}

_UpperCamelCase 			=		{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_UpperCamelCase 			=		{
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}







class   __lowercase      (_UpperCAmelCase ):
 _UpperCamelCase			  =  VOCAB_FILES_NAMES
 _UpperCamelCase			  =  PRETRAINED_VOCAB_FILES_MAP
 _UpperCamelCase			  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _UpperCamelCase			  =  ["""input_ids""", """attention_mask"""]
 def __init__(							self      ,							A_      ,							A_=False      ,							A_=False      ,							A_=False      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_ = None      ,							**A_      ,							)					->None:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					{} if sp_model_kwargs is None else sp_model_kwargs

       __lowerCAmelCase					:						int					=					kwargs.get('''name_or_path'''					)
       if name_or_path is None:
             logger.warning(
                 '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                 ''' you are testing the model, this can safely be ignored'''					)
             __lowerCAmelCase					:						Union[str, Any]					=					'''None'''

       # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
       __lowerCAmelCase					:						str					=					'''<|endoftext|>''' if eos_token is None else eos_token
       __lowerCAmelCase					:						Any					=					'''<unk>''' if unk_token is None else unk_token
       if "gpt-sw3-7b" in name_or_path:
             __lowerCAmelCase					:						Dict					=					unk_token if pad_token is None else pad_token
             __lowerCAmelCase					:						int					=					eos_token if bos_token is None else bos_token
       else:
             __lowerCAmelCase					:						Optional[int]					=					'''<pad>''' if pad_token is None else pad_token
             __lowerCAmelCase					:						List[str]					=					'''<s>''' if bos_token is None else bos_token

       super().__init__(
           do_lower_case=A_      ,							remove_space=A_      ,							keep_accents=A_      ,							bos_token=A_      ,							eos_token=A_      ,							unk_token=A_      ,							pad_token=A_      ,							sp_model_kwargs=self.sp_model_kwargs      ,							**A_      ,							)

       __lowerCAmelCase					:						Union[str, Any]					=					do_lower_case
       __lowerCAmelCase					:						Union[str, Any]					=					remove_space
       __lowerCAmelCase					:						int					=					keep_accents
       __lowerCAmelCase					:						Union[str, Any]					=					vocab_file

       __lowerCAmelCase					:						int					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(A_					)

       # Used for whitespace normalization in input texts
       # fmt : off
       __lowerCAmelCase					:						List[Any]					=					{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
       # fmt : on

       # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
       __lowerCAmelCase					:						int					=					re.compile(
           f"""[{"".join(map(A_      ,							list(range(0      ,							9					)					) + list(range(11      ,							32					)					) + list(range(127      ,							160					)					) + [160, 173, 8203]					)					)}]"""					)
 def __getstate__(							self					)					->Dict:



       '''simple docstring'''





       __lowerCAmelCase					:						Union[str, Any]					=					self.__dict__.copy()
       __lowerCAmelCase					:						List[Any]					=					None
       return state
 def __setstate__(							self      ,							A_					)					->Tuple:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					d

       # for backward compatibility
       if not hasattr(self      ,							'''sp_model_kwargs'''					):
             __lowerCAmelCase					:						List[Any]					=					{}

       __lowerCAmelCase					:						List[str]					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(self.vocab_file					)
 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       return len(self.sp_model					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.non_printing_characters_re.sub(''''''      ,							A_					)

       # Normalize whitespaces
       __lowerCAmelCase					:						List[str]					=					''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]					)

       # NFC Unicode normalization
       __lowerCAmelCase					:						Tuple					=					unicodedata.normalize('''NFC'''      ,							A_					)
       return text
 def 	UpperCamelCase__  (							self      ,							A_      ,							**A_					)					->List[str]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.preprocess_text(A_					)
       return self.sp_model.encode(A_      ,							out_type=A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->int:



       '''simple docstring'''





       return self.sp_model.PieceToId(A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.IdToPiece(A_					)
 @staticmethod
 def 	UpperCamelCase__  (							A_					)					->str:



       '''simple docstring'''





       return out_string
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					[]
       __lowerCAmelCase					:						Tuple					=					''''''
       __lowerCAmelCase					:						int					=					False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                   if not prev_is_special:
                         out_string += " "

                   out_string += self.sp_model.decode(A_					) + token
                   __lowerCAmelCase					:						Optional[Any]					=					True
                   __lowerCAmelCase					:						Optional[int]					=					[]
             else:
                   current_sub_tokens.append(A_					)
                   __lowerCAmelCase					:						str					=					False
       out_string += self.sp_model.decode(A_					)

       return out_string
 def 	UpperCamelCase__  (							self					)					->Dict[str, int]:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					{self.convert_ids_to_tokens(A_					): i for i in range(self.vocab_size					)}
       vocab.update(self.added_tokens_encoder					)
       return vocab
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = None					)					->Tuple[str]:



       '''simple docstring'''





       if not os.path.isdir(A_					):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
             return
       __lowerCAmelCase					:						Any					=					os.path.join(
           A_      ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

       if os.path.abspath(self.vocab_file					) != os.path.abspath(A_					) and os.path.isfile(self.vocab_file					):
             copyfile(self.vocab_file      ,							A_					)
       elif not os.path.isfile(self.vocab_file					):
             with open(A_      ,							'''wb'''					) as fi:
                   __lowerCAmelCase					:						Dict					=					self.sp_model.serialized_model_proto()
                   fi.write(A_					)

       return (out_vocab_file,)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = False					)					->Union[List[int], List[List[int]], "torch.Tensor"]:



       '''simple docstring'''





       if isinstance(A_      ,							A_					):
             __lowerCAmelCase					:						Optional[Any]					=					self.preprocess_text(A_					)
             __lowerCAmelCase					:						Dict					=					self.sp_model.encode(A_					)
       else:
             __lowerCAmelCase					:						Dict					=					[self.preprocess_text(A_					) for t in text]
             __lowerCAmelCase					:						Optional[int]					=					self.sp_model.encode(A_					)

       if return_tensors is True or return_tensors == "pt":
             __lowerCAmelCase					:						Tuple					=					torch.tensor(A_					)

       return token_ids
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.decode(A_					)





 def 	UpperCamelCase__  (							self      ,							A_					)					->List[int]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					[f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
       __lowerCAmelCase					:						Any					=					(
           f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(A_					) + f"""{self.bos_token}Bot:"""
       )
       return self.encode(text=A_					)


