



"""simple docstring"""






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowerCAmelCase__              =							logging.get_logger(__name__)
lowerCAmelCase__              =							{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase__              =							{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

lowerCAmelCase__              =							{
    '''AI-Sweden/gpt-sw3-126m''': 2048,
    '''AI-Sweden/gpt-sw3-350m''': 2048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2048,
    '''AI-Sweden/gpt-sw3-20b''': 2048,
}






class 			__snake_case			(      _a):
  snake_case__							:  Tuple         =		VOCAB_FILES_NAMES
  snake_case__							:  Any         =		PRETRAINED_VOCAB_FILES_MAP
  snake_case__							:  Union[str, Any]         =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case__							:  str         =		["""input_ids""", """attention_mask"""]




  def __init__(					self	:		Optional[int]		,   __lowerCAmelCase	:		Dict		,   __lowerCAmelCase	:		str=False		,   __lowerCAmelCase	:		str=False		,   __lowerCAmelCase	:		Optional[int]=False		,   __lowerCAmelCase	:		Union[str, Any]=None		,   __lowerCAmelCase	:		Union[str, Any]=None		,   __lowerCAmelCase	:		List[Any]=None		,   __lowerCAmelCase	:		Tuple=None		,   __lowerCAmelCase	:		List[str] = None		,   **__lowerCAmelCase	:		int		,   ):






         """simple docstring"""






         _lowerCamelCase		:							Union[str, Any]            =					{} if sp_model_kwargs is None else sp_model_kwargs

         _lowerCamelCase		:							Optional[Any]            =					kwargs.get('''name_or_path'''						)
         if name_or_path is None:
                logger.warning(
                    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                    ''' you are testing the model, this can safely be ignored'''						)
                _lowerCamelCase		:							List[Any]            =					'''None'''

         # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
         _lowerCamelCase		:							Any            =					'''<|endoftext|>''' if eos_token is None else eos_token
         _lowerCamelCase		:							List[Any]            =					'''<unk>''' if unk_token is None else unk_token
         if "gpt-sw3-7b" in name_or_path:
                _lowerCamelCase		:							Optional[Any]            =					unk_token if pad_token is None else pad_token
                _lowerCamelCase		:							str            =					eos_token if bos_token is None else bos_token
         else:
                _lowerCamelCase		:							Optional[Any]            =					'''<pad>''' if pad_token is None else pad_token
                _lowerCamelCase		:							Any            =					'''<s>''' if bos_token is None else bos_token

         super().__init__(
             do_lower_case=__lowerCAmelCase		,   remove_space=__lowerCAmelCase		,   keep_accents=__lowerCAmelCase		,   bos_token=__lowerCAmelCase		,   eos_token=__lowerCAmelCase		,   unk_token=__lowerCAmelCase		,   pad_token=__lowerCAmelCase		,   sp_model_kwargs=self.sp_model_kwargs		,   **__lowerCAmelCase		,   )

         _lowerCamelCase		:							List[Any]            =					do_lower_case
         _lowerCamelCase		:							Union[str, Any]            =					remove_space
         _lowerCamelCase		:							Optional[int]            =					keep_accents
         _lowerCamelCase		:							List[str]            =					vocab_file

         _lowerCamelCase		:							Optional[Any]            =					spm.SentencePieceProcessor(**self.sp_model_kwargs						)
         self.sp_model.Load(__lowerCAmelCase						)

         # Used for whitespace normalization in input texts
         # fmt : off
         _lowerCamelCase		:							Optional[int]            =					{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
         # fmt : on

         # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
         _lowerCamelCase		:							List[Any]            =					re.compile(
             f'''[{"".join(map(__lowerCAmelCase		,   list(range(0		,   9						)						) + list(range(1_1		,   3_2						)						) + list(range(1_2_7		,   1_6_0						)						) + [1_6_0, 1_7_3, 8_2_0_3]						)						)}]'''						)




  def __getstate__(					self	:		Tuple						):






         """simple docstring"""






         _lowerCamelCase		:							Dict            =					self.__dict__.copy()
         _lowerCamelCase		:							Dict            =					None
         return state




  def __setstate__(					self	:		Dict		,   __lowerCAmelCase	:		Dict						):






         """simple docstring"""






         _lowerCamelCase		:							List[str]            =					d

         # for backward compatibility
         if not hasattr(self		,   '''sp_model_kwargs'''						):
                _lowerCamelCase		:							List[Any]            =					{}

         _lowerCamelCase		:							Tuple            =					spm.SentencePieceProcessor(**self.sp_model_kwargs						)
         self.sp_model.Load(self.vocab_file						)




  @property
  # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
  def 		SCREAMING_SNAKE_CASE							(					self	:		Tuple						):






         """simple docstring"""






         return len(self.sp_model						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[Any]		,   __lowerCAmelCase	:		Union[str, Any]						):






         """simple docstring"""






         _lowerCamelCase		:							Dict            =					self.non_printing_characters_re.sub(''''''		,   __lowerCAmelCase						)

         # Normalize whitespaces
         _lowerCamelCase		:							Any            =					''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]						)

         # NFC Unicode normalization
         _lowerCamelCase		:							List[Any]            =					unicodedata.normalize('''NFC'''		,   __lowerCAmelCase						)
         return text




  def 		SCREAMING_SNAKE_CASE							(					self	:		int		,   __lowerCAmelCase	:		int		,   **__lowerCAmelCase	:		List[str]						):






         """simple docstring"""






         _lowerCamelCase		:							str            =					self.preprocess_text(__lowerCAmelCase						)
         return self.sp_model.encode(__lowerCAmelCase		,   out_type=__lowerCAmelCase						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		Tuple		,   __lowerCAmelCase	:		List[str]						):






         """simple docstring"""






         return self.sp_model.PieceToId(__lowerCAmelCase						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		Tuple		,   __lowerCAmelCase	:		Tuple						):






         """simple docstring"""






         return self.sp_model.IdToPiece(__lowerCAmelCase						)




  @staticmethod
  def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase	:		Any						):






         """simple docstring"""






         return out_string




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[Any]		,   __lowerCAmelCase	:		Optional[Any]						):






         """simple docstring"""






         _lowerCamelCase		:							int            =					[]
         _lowerCamelCase		:							Union[str, Any]            =					''''''
         _lowerCamelCase		:							Optional[int]            =					False
         for token in tokens:
                # make sure that special tokens are not decoded using sentencepiece model
                if token in self.all_special_tokens:
                       # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                       if not prev_is_special:
                              out_string += " "

                       out_string += self.sp_model.decode(__lowerCAmelCase						) + token
                       _lowerCamelCase		:							Tuple            =					True
                       _lowerCamelCase		:							str            =					[]
                else:
                       current_sub_tokens.append(__lowerCAmelCase						)
                       _lowerCamelCase		:							List[str]            =					False
         out_string += self.sp_model.decode(__lowerCAmelCase						)

         return out_string




  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict						):






         """simple docstring"""






         _lowerCamelCase		:							int            =					{self.convert_ids_to_tokens(__lowerCAmelCase						): i for i in range(self.vocab_size						)}
         vocab.update(self.added_tokens_encoder						)
         return vocab




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[int]		,   __lowerCAmelCase	:		List[str]		,   __lowerCAmelCase	:		int = None						):






         """simple docstring"""






         if not os.path.isdir(__lowerCAmelCase						):
                logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
                return
         _lowerCamelCase		:							List[str]            =					os.path.join(
             __lowerCAmelCase		,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

         if os.path.abspath(self.vocab_file						) != os.path.abspath(__lowerCAmelCase						) and os.path.isfile(self.vocab_file						):
                copyfile(self.vocab_file		,   __lowerCAmelCase						)
         elif not os.path.isfile(self.vocab_file						):
                with open(__lowerCAmelCase		,   '''wb'''						) as fi:
                       _lowerCamelCase		:							Tuple            =					self.sp_model.serialized_model_proto()
                       fi.write(__lowerCAmelCase						)

         return (out_vocab_file,)




  def 		SCREAMING_SNAKE_CASE							(					self	:		int		,   __lowerCAmelCase	:		Any		,   __lowerCAmelCase	:		Dict = False						):






         """simple docstring"""






         if isinstance(__lowerCAmelCase		,   __lowerCAmelCase						):
                _lowerCamelCase		:							str            =					self.preprocess_text(__lowerCAmelCase						)
                _lowerCamelCase		:							Union[str, Any]            =					self.sp_model.encode(__lowerCAmelCase						)
         else:
                _lowerCamelCase		:							Any            =					[self.preprocess_text(__lowerCAmelCase						) for t in text]
                _lowerCamelCase		:							Dict            =					self.sp_model.encode(__lowerCAmelCase						)

         if return_tensors is True or return_tensors == "pt":
                _lowerCamelCase		:							List[str]            =					torch.tensor(__lowerCAmelCase						)

         return token_ids




  def 		SCREAMING_SNAKE_CASE							(					self	:		int		,   __lowerCAmelCase	:		List[Any]						):






         """simple docstring"""






         return self.sp_model.decode(__lowerCAmelCase						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]		,   __lowerCAmelCase	:		Any						):






         """simple docstring"""






         _lowerCamelCase		:							int            =					[f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
         _lowerCamelCase		:							str            =					(
             f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(__lowerCAmelCase						) + f'''{self.bos_token}Bot:'''
         )
         return self.encode(text=__lowerCAmelCase						)









import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__           =			logging.get_logger(__name__)

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[Any]      =							"""encoder-decoder"""
							snake_case  :   Optional[int]      =							True






							def __init__(    self	,			**__lowerCAmelCase	):
														super().__init__(**__lowerCAmelCase	)
														assert (
														    "encoder" in kwargs and "decoder" in kwargs
														), "Config has to be initialized with encoder and decoder config"
														UpperCamelCase__					  =      kwargs.pop("""encoder"""	)
														UpperCamelCase__					  =      encoder_config.pop("""model_type"""	)
														UpperCamelCase__					  =      kwargs.pop("""decoder"""	)
														UpperCamelCase__					  =      decoder_config.pop("""model_type"""	)

														from ..auto.configuration_auto import AutoConfig

														UpperCamelCase__					  =      AutoConfig.for_model(__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      AutoConfig.for_model(__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      True






							@classmethod
							def 					_lowerCamelCase		(    cls	,			__lowerCAmelCase	,			__lowerCAmelCase	,			**__lowerCAmelCase	):
														logger.info("""Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config"""	)
														UpperCamelCase__					  =      True
														UpperCamelCase__					  =      True

														return cls(encoder=encoder_config.to_dict()	,			decoder=decoder_config.to_dict()	,			**__lowerCAmelCase	)







							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      copy.deepcopy(self.__dict__	)
														UpperCamelCase__					  =      self.encoder.to_dict()
														UpperCamelCase__					  =      self.decoder.to_dict()
														UpperCamelCase__					  =      self.__class__.model_type
														return output




