



import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
						import torch


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


A__  :  Any								=					logging.get_logger(__name__)
A__  :  int								=					{'''vocab_file''': '''spiece.model'''}

A__  :  Optional[Any]								=					{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

A__  :  int								=					{
    '''AI-Sweden/gpt-sw3-126m''': 2048,
    '''AI-Sweden/gpt-sw3-350m''': 2048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2048,
    '''AI-Sweden/gpt-sw3-20b''': 2048,
}



class 				__snake_case						(     UpperCamelCase_						):
							_a			      = VOCAB_FILES_NAMES
							_a			      = PRETRAINED_VOCAB_FILES_MAP
							_a			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_a			      = ['''input_ids''', '''attention_mask''']




							def __init__(       self     :       List[str]				,				A_     :       Tuple				,				A_     :       Union[str, Any]=False				,				A_     :       Optional[int]=False				,				A_     :       Union[str, Any]=False				,				A_     :       Optional[Any]=None				,				A_     :       str=None				,				A_     :       Any=None				,				A_     :       Any=None				,				A_     :       Optional[Dict[str, Any]] = None				,				**A_     :       Optional[int]				,				):
									lowerCAmelCase_			:     Tuple									= {} if sp_model_kwargs is None else sp_model_kwargs

									lowerCAmelCase_			:     List[str]									= kwargs.get('''name_or_path''')
									if name_or_path is None:
											logger.warning(
											    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
											    ''' you are testing the model, this can safely be ignored''')
											lowerCAmelCase_			:     Optional[Any]									= '''None'''

									# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
									lowerCAmelCase_			:     List[Any]									= '''<|endoftext|>''' if eos_token is None else eos_token
									lowerCAmelCase_			:     int									= '''<unk>''' if unk_token is None else unk_token
									if "gpt-sw3-7b" in name_or_path:
											lowerCAmelCase_			:     str									= unk_token if pad_token is None else pad_token
											lowerCAmelCase_			:     Tuple									= eos_token if bos_token is None else bos_token
									else:
											lowerCAmelCase_			:     List[Any]									= '''<pad>''' if pad_token is None else pad_token
											lowerCAmelCase_			:     Optional[int]									= '''<s>''' if bos_token is None else bos_token

									super().__init__(
									    do_lower_case=A_				,				remove_space=A_				,				keep_accents=A_				,				bos_token=A_				,				eos_token=A_				,				unk_token=A_				,				pad_token=A_				,				sp_model_kwargs=self.sp_model_kwargs				,				**A_				,				)

									lowerCAmelCase_			:     Union[str, Any]									= do_lower_case
									lowerCAmelCase_			:     Any									= remove_space
									lowerCAmelCase_			:     str									= keep_accents
									lowerCAmelCase_			:     Optional[int]									= vocab_file

									lowerCAmelCase_			:     List[Any]									= spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(A_)

									# Used for whitespace normalization in input texts
									# fmt : off
									lowerCAmelCase_			:     Dict									= {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
									# fmt : on

									# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
									lowerCAmelCase_			:     Tuple									= re.compile(
									    F"""[{"".join(map(A_				,				list(range(0				,				9)) + list(range(1_1				,				3_2)) + list(range(1_2_7				,				1_6_0)) + [1_6_0, 1_7_3, 8_2_0_3]))}]""")




							def __getstate__(       self     :       Optional[int]):
									lowerCAmelCase_			:     str									= self.__dict__.copy()
									lowerCAmelCase_			:     str									= None
									return state




							def __setstate__(       self     :       str				,				A_     :       List[Any]):
									lowerCAmelCase_			:     Optional[int]									= d

									# for backward compatibility
									if not hasattr(self				,				'''sp_model_kwargs'''):
											lowerCAmelCase_			:     Optional[Any]									= {}

									lowerCAmelCase_			:     str									= spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(self.vocab_file)




							@property
							# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
							def    UpperCAmelCase__			(       self     :       Optional[Any]):
									return len(self.sp_model)




							def    UpperCAmelCase__			(       self     :       Union[str, Any]				,				A_     :       str):
									lowerCAmelCase_			:     Optional[Any]									= self.non_printing_characters_re.sub(''''''				,				A_)

									# Normalize whitespaces
									lowerCAmelCase_			:     Union[str, Any]									= ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text])

									# NFC Unicode normalization
									lowerCAmelCase_			:     List[str]									= unicodedata.normalize('''NFC'''				,				A_)
									return text




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       str				,				**A_     :       List[Any]):
									lowerCAmelCase_			:     str									= self.preprocess_text(A_)
									return self.sp_model.encode(A_				,				out_type=A_)




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       str):
									return self.sp_model.PieceToId(A_)




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       int):
									return self.sp_model.IdToPiece(A_)




							@staticmethod
							def    UpperCAmelCase__			(       A_     :       str):
									return out_string




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       List[str]):
									lowerCAmelCase_			:     Optional[int]									= []
									lowerCAmelCase_			:     Optional[int]									= ''''''
									lowerCAmelCase_			:     Dict									= False
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
													if not prev_is_special:
															out_string += " "

													out_string += self.sp_model.decode(A_) + token
													lowerCAmelCase_			:     str									= True
													lowerCAmelCase_			:     Optional[int]									= []
											else:
													current_sub_tokens.append(A_)
													lowerCAmelCase_			:     List[Any]									= False
									out_string += self.sp_model.decode(A_)

									return out_string




							def    UpperCAmelCase__			(       self     :       int):
									lowerCAmelCase_			:     Optional[Any]									= {self.convert_ids_to_tokens(A_): i for i in range(self.vocab_size)}
									vocab.update(self.added_tokens_encoder)
									return vocab




							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       str				,				A_     :       Optional[str] = None):
									if not os.path.isdir(A_):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
											return
									lowerCAmelCase_			:     Tuple									= os.path.join(
									    A_				,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

									if os.path.abspath(self.vocab_file) != os.path.abspath(A_) and os.path.isfile(self.vocab_file):
											copyfile(self.vocab_file				,				A_)
									elif not os.path.isfile(self.vocab_file):
											with open(A_				,				'''wb''') as fi:
													lowerCAmelCase_			:     Tuple									= self.sp_model.serialized_model_proto()
													fi.write(A_)

									return (out_vocab_file,)




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       Union[str, List[str]]				,				A_     :       Union[str, bool] = False):

									if isinstance(A_				,				A_):
											lowerCAmelCase_			:     int									= self.preprocess_text(A_)
											lowerCAmelCase_			:     List[str]									= self.sp_model.encode(A_)
									else:
											lowerCAmelCase_			:     Optional[Any]									= [self.preprocess_text(A_) for t in text]
											lowerCAmelCase_			:     List[Any]									= self.sp_model.encode(A_)

									if return_tensors is True or return_tensors == "pt":
											lowerCAmelCase_			:     Tuple									= torch.tensor(A_)

									return token_ids




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       Union[int, List[int]]):
									return self.sp_model.decode(A_)







							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       "Conversation"):
									lowerCAmelCase_			:     Union[str, Any]									= [F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
									lowerCAmelCase_			:     Tuple									= (
									    F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(A_) + F"""{self.bos_token}Bot:"""
									)
									return self.encode(text=A_)










from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A__  :  Union[str, Any]								=					logging.get_logger(__name__)

A__  :  Tuple								=					{
    '''facebook/xlm-roberta-xl''': '''https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json''',
    '''facebook/xlm-roberta-xxl''': '''https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json''',
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}



class 				__snake_case						(     UpperCamelCase_						):
							_a			      = '''xlm-roberta-xl'''




							def __init__(       self     :       int				,				A_     :       List[str]=2_5_0_8_8_0				,				A_     :       List[str]=2_5_6_0				,				A_     :       Optional[int]=3_6				,				A_     :       List[Any]=3_2				,				A_     :       Optional[int]=1_0_2_4_0				,				A_     :       Dict="gelu"				,				A_     :       int=0.1				,				A_     :       Optional[Any]=0.1				,				A_     :       int=5_1_4				,				A_     :       Any=1				,				A_     :       Optional[Any]=0.02				,				A_     :       str=1e-05				,				A_     :       Dict=1				,				A_     :       Any=0				,				A_     :       Tuple=2				,				A_     :       str="absolute"				,				A_     :       str=True				,				A_     :       List[str]=None				,				**A_     :       Dict				,				):
									super().__init__(pad_token_id=A_				,				bos_token_id=A_				,				eos_token_id=A_				,				**A_)
									lowerCAmelCase_			:     Tuple									= vocab_size
									lowerCAmelCase_			:     List[str]									= hidden_size
									lowerCAmelCase_			:     int									= num_hidden_layers
									lowerCAmelCase_			:     int									= num_attention_heads
									lowerCAmelCase_			:     Dict									= hidden_act
									lowerCAmelCase_			:     int									= intermediate_size
									lowerCAmelCase_			:     Tuple									= hidden_dropout_prob
									lowerCAmelCase_			:     Optional[Any]									= attention_probs_dropout_prob
									lowerCAmelCase_			:     Union[str, Any]									= max_position_embeddings
									lowerCAmelCase_			:     Dict									= type_vocab_size
									lowerCAmelCase_			:     str									= initializer_range
									lowerCAmelCase_			:     str									= layer_norm_eps
									lowerCAmelCase_			:     Optional[Any]									= position_embedding_type
									lowerCAmelCase_			:     Optional[Any]									= use_cache
									lowerCAmelCase_			:     List[str]									= classifier_dropout





class 				__snake_case						(     UpperCamelCase_						):




							@property
							def    UpperCAmelCase__			(       self     :       List[str]):
									if self.task == "multiple-choice":
											lowerCAmelCase_			:     Union[str, Any]									= {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
									else:
											lowerCAmelCase_			:     List[Any]									= {0: '''batch''', 1: '''sequence'''}
									return OrderedDict(
									    [
									        ('''input_ids''', dynamic_axis),
									        ('''attention_mask''', dynamic_axis),
									    ])





