





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
	import torch


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


lowerCAmelCase_    =				logging.get_logger(__name__)
lowerCAmelCase_    =				{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase_    =				{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

lowerCAmelCase_    =				{
    '''AI-Sweden/gpt-sw3-126m''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-350m''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-1.6b''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-6.7b''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-20b''': 2_0_4_8,
}



class 					__lowerCAmelCase       (			_a     ):
			lowerCamelCase_     :     List[Any]       			=      VOCAB_FILES_NAMES
			lowerCamelCase_     :     Optional[int]       			=      PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase_     :     List[Any]       			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase_     :     Tuple       			=      ['''input_ids''', '''attention_mask''']






			def __init__(self					,							__magic_name__					,							__magic_name__=False					,							__magic_name__=False					,							__magic_name__=False					,							__magic_name__=None					,							__magic_name__=None					,							__magic_name__=None					,							__magic_name__=None					,							__magic_name__ = None					,							**__magic_name__					,							)     ->   None:


							'''simple docstring'''



							snake_case_						:    List[str]				=  {} if sp_model_kwargs is None else sp_model_kwargs

							snake_case_						:    Union[str, Any]				=  kwargs.get('''name_or_path'''    )
							if name_or_path is None:
											logger.warning(
											    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
											    ''' you are testing the model, this can safely be ignored'''    )
											snake_case_						:    Dict				=  '''None'''

							# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
							snake_case_						:    str				=  '''<|endoftext|>''' if eos_token is None else eos_token
							snake_case_						:    str				=  '''<unk>''' if unk_token is None else unk_token
							if "gpt-sw3-7b" in name_or_path:
											snake_case_						:    Dict				=  unk_token if pad_token is None else pad_token
											snake_case_						:    List[str]				=  eos_token if bos_token is None else bos_token
							else:
											snake_case_						:    Tuple				=  '''<pad>''' if pad_token is None else pad_token
											snake_case_						:    Optional[int]				=  '''<s>''' if bos_token is None else bos_token

							super().__init__(
							    do_lower_case=__magic_name__					,							remove_space=__magic_name__					,							keep_accents=__magic_name__					,							bos_token=__magic_name__					,							eos_token=__magic_name__					,							unk_token=__magic_name__					,							pad_token=__magic_name__					,							sp_model_kwargs=self.sp_model_kwargs					,							**__magic_name__					,							)

							snake_case_						:    Dict				=  do_lower_case
							snake_case_						:    Optional[Any]				=  remove_space
							snake_case_						:    Optional[Any]				=  keep_accents
							snake_case_						:    Tuple				=  vocab_file

							snake_case_						:    Dict				=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(__magic_name__    )

							# Used for whitespace normalization in input texts
							# fmt : off
							snake_case_						:    List[Any]				=  {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
							# fmt : on

							# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
							snake_case_						:    str				=  re.compile(
							    F'''[{"".join(map(__magic_name__					,							list(range(0					,							9    )    ) + list(range(11					,							32    )    ) + list(range(127					,							160    )    ) + [160, 173, 8203]    )    )}]'''    )






			def __getstate__(self    )     ->   Tuple:


							'''simple docstring'''



							snake_case_						:    List[str]				=  self.__dict__.copy()
							snake_case_						:    Tuple				=  None
							return state






			def __setstate__(self					,							__magic_name__    )     ->   Optional[int]:


							'''simple docstring'''



							snake_case_						:    Optional[int]				=  d

							# for backward compatibility
							if not hasattr(self					,							'''sp_model_kwargs'''    ):
											snake_case_						:    Tuple				=  {}

							snake_case_						:    Any				=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(self.vocab_file    )






			@property
			# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
			def 			lowerCamelCase  (self    )     ->   int:


							'''simple docstring'''



							return len(self.sp_model    )






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   str:


							'''simple docstring'''



							snake_case_						:    Dict				=  self.non_printing_characters_re.sub(''''''					,							__magic_name__    )

							# Normalize whitespaces
							snake_case_						:    Optional[int]				=  ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]    )

							# NFC Unicode normalization
							snake_case_						:    Union[str, Any]				=  unicodedata.normalize('''NFC'''					,							__magic_name__    )
							return text






			def 			lowerCamelCase  (self					,							__magic_name__					,							**__magic_name__    )     ->   List[str]:


							'''simple docstring'''



							snake_case_						:    Dict				=  self.preprocess_text(__magic_name__    )
							return self.sp_model.encode(__magic_name__					,							out_type=__magic_name__    )






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   int:


							'''simple docstring'''



							return self.sp_model.PieceToId(__magic_name__    )






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   str:


							'''simple docstring'''



							return self.sp_model.IdToPiece(__magic_name__    )






			@staticmethod
			def 			lowerCamelCase  (__magic_name__    )     ->   str:


							'''simple docstring'''



							return out_string






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   str:


							'''simple docstring'''



							snake_case_						:    Union[str, Any]				=  []
							snake_case_						:    Dict				=  ''''''
							snake_case_						:    int				=  False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
															if not prev_is_special:
																			out_string += " "

															out_string += self.sp_model.decode(__magic_name__    ) + token
															snake_case_						:    Dict				=  True
															snake_case_						:    List[Any]				=  []
											else:
															current_sub_tokens.append(__magic_name__    )
															snake_case_						:    Dict				=  False
							out_string += self.sp_model.decode(__magic_name__    )

							return out_string






			def 			lowerCamelCase  (self    )     ->   Dict[str, int]:


							'''simple docstring'''



							snake_case_						:    str				=  {self.convert_ids_to_tokens(__magic_name__    ): i for i in range(self.vocab_size    )}
							vocab.update(self.added_tokens_encoder    )
							return vocab






			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__ = None    )     ->   Tuple[str]:


							'''simple docstring'''



							if not os.path.isdir(__magic_name__    ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
											return
							snake_case_						:    List[str]				=  os.path.join(
							    __magic_name__					,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

							if os.path.abspath(self.vocab_file    ) != os.path.abspath(__magic_name__    ) and os.path.isfile(self.vocab_file    ):
											copyfile(self.vocab_file					,							__magic_name__    )
							elif not os.path.isfile(self.vocab_file    ):
											with open(__magic_name__					,							'''wb'''    ) as fi:
															snake_case_						:    Union[str, Any]				=  self.sp_model.serialized_model_proto()
															fi.write(__magic_name__    )

							return (out_vocab_file,)






			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__ = False    )     ->   Union[List[int], List[List[int]], "torch.Tensor"]:


							'''simple docstring'''



							if isinstance(__magic_name__					,							__magic_name__    ):
											snake_case_						:    List[str]				=  self.preprocess_text(__magic_name__    )
											snake_case_						:    List[str]				=  self.sp_model.encode(__magic_name__    )
							else:
											snake_case_						:    Dict				=  [self.preprocess_text(__magic_name__    ) for t in text]
											snake_case_						:    Union[str, Any]				=  self.sp_model.encode(__magic_name__    )

							if return_tensors is True or return_tensors == "pt":
											snake_case_						:    List[Any]				=  torch.tensor(__magic_name__    )

							return token_ids






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   str:


							'''simple docstring'''



							return self.sp_model.decode(__magic_name__    )






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   List[int]:


							'''simple docstring'''



							snake_case_						:    Dict				=  [F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
							snake_case_						:    Any				=  (
							    F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(__magic_name__    ) + F'''{self.bos_token}Bot:'''
							)
							return self.encode(text=__magic_name__    )







from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
	if not (is_transformers_available() and is_torch_available() and is_transformers_version('''>=''', '''4.25.0''')):
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
	from .pipeline_unclip import UnCLIPPipeline
	from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
	from .text_proj import UnCLIPTextProjModel
