'''simple docstring'''




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowercase          =  logging.get_logger(__name__)

__lowercase          =  '▁'

__lowercase          =  {'vocab_file': 'sentencepiece.bpe.model'}

__lowercase          =  {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__lowercase          =  {
    'xlm-roberta-base': 5_1_2,
    'xlm-roberta-large': 5_1_2,
    'xlm-roberta-large-finetuned-conll02-dutch': 5_1_2,
    'xlm-roberta-large-finetuned-conll02-spanish': 5_1_2,
    'xlm-roberta-large-finetuned-conll03-english': 5_1_2,
    'xlm-roberta-large-finetuned-conll03-german': 5_1_2,
}






class 				a__(					lowerCAmelCase__	):


			'''simple docstring'''






			UpperCAmelCase_			:     List[Any]      =       VOCAB_FILES_NAMES
			UpperCAmelCase_			:     Tuple      =       PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_			:     List[Any]      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_			:     List[str]      =       ['''input_ids''', '''attention_mask''']



			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase="<s>"       ,    __lowerCAmelCase="</s>"       ,    __lowerCAmelCase="</s>"       ,    __lowerCAmelCase="<s>"       ,    __lowerCAmelCase="<unk>"       ,    __lowerCAmelCase="<pad>"       ,    __lowerCAmelCase="<mask>"       ,    __lowerCAmelCase = None       ,    **__lowerCAmelCase       ,    ):




					"""simple docstring"""


					lowerCAmelCase   					=  AddedToken(__lowerCAmelCase       ,    lstrip=__lowerCAmelCase       ,    rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,    __lowerCAmelCase) else mask_token

					lowerCAmelCase   					=  {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    bos_token=__lowerCAmelCase       ,    eos_token=__lowerCAmelCase       ,    unk_token=__lowerCAmelCase       ,    sep_token=__lowerCAmelCase       ,    cls_token=__lowerCAmelCase       ,    pad_token=__lowerCAmelCase       ,    mask_token=__lowerCAmelCase       ,    sp_model_kwargs=self.sp_model_kwargs       ,    **__lowerCAmelCase       ,    )

					lowerCAmelCase   					=  spm.SentencePieceProcessor(**self.sp_model_kwargs)
					self.sp_model.Load(str(__lowerCAmelCase))
					lowerCAmelCase   					=  vocab_file

					# Original fairseq vocab and spm vocab must be "aligned":
					# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
					# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
					# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
					# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

					# Mimic fairseq token-to-id alignment for the first 4 token
					lowerCAmelCase   					=  {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

					# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
					lowerCAmelCase   					=  1

					lowerCAmelCase   					=  len(self.sp_model) + self.fairseq_offset
					lowerCAmelCase   					=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}



			def __getstate__(   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.__dict__.copy()
					lowerCAmelCase   					=  None
					lowerCAmelCase   					=  self.sp_model.serialized_model_proto()
					return state



			def __setstate__(   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  d

					# for backward compatibility
					if not hasattr(self       ,    """sp_model_kwargs"""):
							lowerCAmelCase   					=  {}

					lowerCAmelCase   					=  spm.SentencePieceProcessor(**self.sp_model_kwargs)
					self.sp_model.LoadFromSerializedProto(self.sp_model_proto)



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None):




					"""simple docstring"""


					if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					lowerCAmelCase   					=  [self.cls_token_id]
					lowerCAmelCase   					=  [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = False):




					"""simple docstring"""


					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=__lowerCAmelCase       ,    token_ids_a=__lowerCAmelCase       ,    already_has_special_tokens=__lowerCAmelCase)

					if token_ids_a is None:
							return [1] + ([0] * len(__lowerCAmelCase)) + [1]
					return [1] + ([0] * len(__lowerCAmelCase)) + [1, 1] + ([0] * len(__lowerCAmelCase)) + [1]



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None):




					"""simple docstring"""


					lowerCAmelCase   					=  [self.sep_token_id]
					lowerCAmelCase   					=  [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



			@property
			def 							a_       (   self):




					"""simple docstring"""


					return len(self.sp_model) + self.fairseq_offset + 1  # Add the <mask> token



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  {self.convert_ids_to_tokens(__lowerCAmelCase): i for i in range(self.vocab_size)}
					vocab.update(self.added_tokens_encoder)
					return vocab



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					return self.sp_model.encode(__lowerCAmelCase       ,    out_type=__lowerCAmelCase)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
					lowerCAmelCase   					=  self.sp_model.PieceToId(__lowerCAmelCase)

					# Need to return unknown token if the SP model returned 0
					return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
					return self.sp_model.IdToPiece(index - self.fairseq_offset)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  """""".join(__lowerCAmelCase).replace(__lowerCAmelCase       ,    """ """).strip()
					return out_string







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None):




					"""simple docstring"""


					if not os.path.isdir(__lowerCAmelCase):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory")
							return
					lowerCAmelCase   					=  os.path.join(
					    __lowerCAmelCase       ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

					if os.path.abspath(self.vocab_file) != os.path.abspath(__lowerCAmelCase) and os.path.isfile(self.vocab_file):
							copyfile(self.vocab_file       ,    __lowerCAmelCase)
					elif not os.path.isfile(self.vocab_file):
							with open(__lowerCAmelCase       ,    """wb""") as fi:
									lowerCAmelCase   					=  self.sp_model.serialized_model_proto()
									fi.write(__lowerCAmelCase)

					return (out_vocab_file,)








"""simple docstring"""






from importlib import import_module

from .logging import get_logger


UpperCAmelCase    :     Any   						=		get_logger(__name__)





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Optional[int] ,       UpperCamelCase    : List[Any] ,       UpperCamelCase    : int=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					attrs or []
												if module is not None:
																			for key in module.__dict__:
																										if key in attrs or not key.startswith("""__"""  ):
																																	setattr(self ,       UpperCamelCase ,       getattr(UpperCamelCase ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						Any      							=					module._original_module if isinstance(UpperCamelCase ,       _PatchedModuleObj  ) else module






class      lowerCamelCase__   :







					"""simple docstring"""


					__a	       =							[]






					def __init__(       self    : str ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : str ,       UpperCamelCase    : List[str] ,       UpperCamelCase    : Optional[Any]=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					obj
												__UpperCAmelCase   :						Union[str, Any]      							=					target
												__UpperCAmelCase   :						List[str]      							=					new
												__UpperCAmelCase   :						Optional[int]      							=					target.split("""."""  )[0]
												__UpperCAmelCase   :						Tuple      							=					{}
												__UpperCAmelCase   :						Union[str, Any]      							=					attrs or []






					def __enter__(       self    : Dict  ):
												'''simple docstring'''






												*__UpperCAmelCase							,__UpperCAmelCase   :						str      							=					self.target.split("""."""  )

												# Patch modules:
												# it's used to patch attributes of submodules like "os.path.join";
												# in this case we need to patch "os" and "os.path"

												for i in range(len(UpperCamelCase  )  ):
																			try:
																										__UpperCAmelCase   :						List[Any]      							=					import_module(""".""".join(submodules[: i + 1]  )  )
																			except ModuleNotFoundError:
																										continue
																			# We iterate over all the globals in self.obj in case we find "os" or "os.path"
																			for attr in self.obj.__dir__():
																										__UpperCAmelCase   :						List[Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																										# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
																										# This allows to patch renamed modules like "from os import path as ospath".
																										if obj_attr is submodule or (
																										    (isinstance(UpperCamelCase ,       _PatchedModuleObj  ) and obj_attr._original_module is submodule)
																										):
																																	__UpperCAmelCase   :						Tuple      							=					obj_attr
																																	# patch at top level
																																	setattr(self.obj ,       UpperCamelCase ,       _PatchedModuleObj(UpperCamelCase ,       attrs=self.attrs  )  )
																																	__UpperCAmelCase   :						int      							=					getattr(self.obj ,       UpperCamelCase  )
																																	# construct lower levels patches
																																	for key in submodules[i + 1 :]:
																																								setattr(UpperCamelCase ,       UpperCamelCase ,       _PatchedModuleObj(getattr(UpperCamelCase ,       UpperCamelCase ,       UpperCamelCase  ) ,       attrs=self.attrs  )  )
																																								__UpperCAmelCase   :						Optional[int]      							=					getattr(UpperCamelCase ,       UpperCamelCase  )
																																	# finally set the target attribute
																																	setattr(UpperCamelCase ,       UpperCamelCase ,       self.new  )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

												if submodules:  # if it's an attribute of a submodule like "os.path.join"
																			try:
																										__UpperCAmelCase   :						int      							=					getattr(import_module(""".""".join(UpperCamelCase  )  ) ,       UpperCamelCase  )
																			except (AttributeError, ModuleNotFoundError):
																										return
																			# We iterate over all the globals in self.obj in case we find "os.path.join"
																			for attr in self.obj.__dir__():
																										# We don't check for the name of the global, but rather if its value *is* "os.path.join".
																										# This allows to patch renamed attributes like "from os.path import join as pjoin".
																										if getattr(self.obj ,       UpperCamelCase  ) is attr_value:
																																	__UpperCAmelCase   :						Union[str, Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																																	setattr(self.obj ,       UpperCamelCase ,       self.new  )
												elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
																			__UpperCAmelCase   :						str      							=					globals()["""__builtins__"""][target_attr]
																			setattr(self.obj ,       UpperCamelCase ,       self.new  )
												else:
																			raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''  )






					def __exit__(       self    : str ,       *UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												for attr in list(self.original  ):
																			setattr(self.obj ,       UpperCamelCase ,       self.original.pop(UpperCamelCase  )  )






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												self.__enter__()
												self._active_patches.append(self  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												try:
																			self._active_patches.remove(self  )
												except ValueError:
																			# If the patch hasn't been started this will fail
																			return None

												return self.__exit__()
