


"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase    :     Union[str, Any]   						=		logging.get_logger(__name__)

UpperCAmelCase    :     List[Any]   						=		{'vocab_file': 'sentencepiece.bpe.model'}

UpperCAmelCase    :     Optional[int]   						=		{
    'vocab_file': {
        'camembert-base': 'https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model',
    }
}

UpperCAmelCase    :     Tuple   						=		{
    'camembert-base': 512,
}

UpperCAmelCase    :     Optional[int]   						=		'▁'





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					__a	       =							VOCAB_FILES_NAMES
					__a	       =							PRETRAINED_VOCAB_FILES_MAP
					__a	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__a	       =							["""input_ids""", """attention_mask"""]






					def __init__(       self    : Any ,       UpperCamelCase    : Optional[int] ,       UpperCamelCase    : List[str]="<s>" ,       UpperCamelCase    : Dict="</s>" ,       UpperCamelCase    : List[str]="</s>" ,       UpperCamelCase    : List[Any]="<s>" ,       UpperCamelCase    : str="<unk>" ,       UpperCamelCase    : List[str]="<pad>" ,       UpperCamelCase    : Optional[Any]="<mask>" ,       UpperCamelCase    : Optional[int]=["<s>NOTUSED", "</s>NOTUSED"] ,       UpperCamelCase    : Optional[Dict[str, Any]] = None ,       **UpperCamelCase    : List[Any] ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else mask_token

												__UpperCAmelCase   :						Optional[int]      							=					{} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    bos_token=UpperCamelCase ,       eos_token=UpperCamelCase ,       unk_token=UpperCamelCase ,       sep_token=UpperCamelCase ,       cls_token=UpperCamelCase ,       pad_token=UpperCamelCase ,       mask_token=UpperCamelCase ,       additional_special_tokens=UpperCamelCase ,       sp_model_kwargs=self.sp_model_kwargs ,       **UpperCamelCase ,       )
												__UpperCAmelCase   :						List[Any]      							=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
												self.sp_model.Load(str(UpperCamelCase  )  )
												__UpperCAmelCase   :						str      							=					vocab_file
												# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
												# sentencepiece vocabulary (this is the case for <s> and </s>
												__UpperCAmelCase   :						Any      							=					{"""<s>NOTUSED""": 0, """<pad>""": 1, """</s>NOTUSED""": 2, """<unk>""": 3}
												__UpperCAmelCase   :						int      							=					len(self.fairseq_tokens_to_ids  )
												__UpperCAmelCase   :						int      							=					len(self.sp_model  ) + len(self.fairseq_tokens_to_ids  )
												__UpperCAmelCase   :						Tuple      							=					{v: k for k, v in self.fairseq_tokens_to_ids.items()}






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None  ):
												'''simple docstring'''






												if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												__UpperCAmelCase   :						List[Any]      							=					[self.cls_token_id]
												__UpperCAmelCase   :						Any      							=					[self.sep_token_id]
												return cls + token_ids_a + sep + sep + token_ids_a + sep






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None ,       UpperCamelCase    : bool = False  ):
												'''simple docstring'''






												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=UpperCamelCase ,       token_ids_a=UpperCamelCase ,       already_has_special_tokens=UpperCamelCase  )

												if token_ids_a is None:
																			return [1] + ([0] * len(UpperCamelCase  )) + [1]
												return [1] + ([0] * len(UpperCamelCase  )) + [1, 1] + ([0] * len(UpperCamelCase  )) + [1]






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					[self.sep_token_id]
												__UpperCAmelCase   :						List[str]      							=					[self.cls_token_id]

												if token_ids_a is None:
																			return len(cls + token_ids_a + sep  ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]






					@property
					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												return len(self.fairseq_tokens_to_ids  ) + len(self.sp_model  )






					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					{self.convert_ids_to_tokens(UpperCamelCase  ): i for i in range(self.vocab_size  )}
												vocab.update(self.added_tokens_encoder  )
												return vocab






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												return self.sp_model.encode(UpperCamelCase ,       out_type=UpperCamelCase  )






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : Dict  ):
												'''simple docstring'''






												if token in self.fairseq_tokens_to_ids:
																			return self.fairseq_tokens_to_ids[token]
												elif self.sp_model.PieceToId(UpperCamelCase  ) == 0:
																			# Convert sentence piece unk token to fairseq unk token index
																			return self.unk_token_id
												return self.fairseq_offset + self.sp_model.PieceToId(UpperCamelCase  )






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												if index in self.fairseq_ids_to_tokens:
																			return self.fairseq_ids_to_tokens[index]
												return self.sp_model.IdToPiece(index - self.fairseq_offset  )






					def lowerCamelCase__							(       self    : str ,       UpperCamelCase    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					[]
												__UpperCAmelCase   :						List[str]      							=					""""""
												__UpperCAmelCase   :						List[str]      							=					False
												for token in tokens:
																			# make sure that special tokens are not decoded using sentencepiece model
																			if token in self.all_special_tokens:
																										if not prev_is_special:
																																	out_string += " "
																										out_string += self.sp_model.decode(UpperCamelCase  ) + token
																										__UpperCAmelCase   :						List[Any]      							=					True
																										__UpperCAmelCase   :						Dict      							=					[]
																			else:
																										current_sub_tokens.append(UpperCamelCase  )
																										__UpperCAmelCase   :						List[Any]      							=					False
												out_string += self.sp_model.decode(UpperCamelCase  )
												return out_string.strip()






					def __getstate__(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					self.__dict__.copy()
												__UpperCAmelCase   :						Dict      							=					None
												return state






					def __setstate__(       self    : Tuple ,       UpperCamelCase    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					d

												# for backward compatibility
												if not hasattr(self ,       """sp_model_kwargs"""  ):
																			__UpperCAmelCase   :						List[Any]      							=					{}

												__UpperCAmelCase   :						Optional[Any]      							=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
												self.sp_model.Load(self.vocab_file  )






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : str ,       UpperCamelCase    : Optional[str] = None  ):
												'''simple docstring'''






												if not os.path.isdir(UpperCamelCase  ):
																			logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
																			return
												__UpperCAmelCase   :						Tuple      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

												if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
																			copyfile(self.vocab_file ,       UpperCamelCase  )
												elif not os.path.isfile(self.vocab_file  ):
																			with open(UpperCamelCase ,       """wb"""  ) as fi:
																										__UpperCAmelCase   :						Optional[Any]      							=					self.sp_model.serialized_model_proto()
																										fi.write(UpperCamelCase  )

												return (out_vocab_file,)




"""simple docstring"""






from importlib import import_module

from .logging import get_logger


UpperCAmelCase    :     Any   						=		get_logger(__name__)





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Optional[int] ,       UpperCamelCase    : List[Any] ,       UpperCamelCase    : int=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					attrs or []
												if module is not None:
																			for key in module.__dict__:
																										if key in attrs or not key.startswith("""__"""  ):
																																	setattr(self ,       UpperCamelCase ,       getattr(UpperCamelCase ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						Any      							=					module._original_module if isinstance(UpperCamelCase ,       _PatchedModuleObj  ) else module






class      lowerCamelCase__   :







					"""simple docstring"""


					__a	       =							[]






					def __init__(       self    : str ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : str ,       UpperCamelCase    : List[str] ,       UpperCamelCase    : Optional[Any]=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					obj
												__UpperCAmelCase   :						Union[str, Any]      							=					target
												__UpperCAmelCase   :						List[str]      							=					new
												__UpperCAmelCase   :						Optional[int]      							=					target.split("""."""  )[0]
												__UpperCAmelCase   :						Tuple      							=					{}
												__UpperCAmelCase   :						Union[str, Any]      							=					attrs or []






					def __enter__(       self    : Dict  ):
												'''simple docstring'''






												*__UpperCAmelCase							,__UpperCAmelCase   :						str      							=					self.target.split("""."""  )

												# Patch modules:
												# it's used to patch attributes of submodules like "os.path.join";
												# in this case we need to patch "os" and "os.path"

												for i in range(len(UpperCamelCase  )  ):
																			try:
																										__UpperCAmelCase   :						List[Any]      							=					import_module(""".""".join(submodules[: i + 1]  )  )
																			except ModuleNotFoundError:
																										continue
																			# We iterate over all the globals in self.obj in case we find "os" or "os.path"
																			for attr in self.obj.__dir__():
																										__UpperCAmelCase   :						List[Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																										# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
																										# This allows to patch renamed modules like "from os import path as ospath".
																										if obj_attr is submodule or (
																										    (isinstance(UpperCamelCase ,       _PatchedModuleObj  ) and obj_attr._original_module is submodule)
																										):
																																	__UpperCAmelCase   :						Tuple      							=					obj_attr
																																	# patch at top level
																																	setattr(self.obj ,       UpperCamelCase ,       _PatchedModuleObj(UpperCamelCase ,       attrs=self.attrs  )  )
																																	__UpperCAmelCase   :						int      							=					getattr(self.obj ,       UpperCamelCase  )
																																	# construct lower levels patches
																																	for key in submodules[i + 1 :]:
																																								setattr(UpperCamelCase ,       UpperCamelCase ,       _PatchedModuleObj(getattr(UpperCamelCase ,       UpperCamelCase ,       UpperCamelCase  ) ,       attrs=self.attrs  )  )
																																								__UpperCAmelCase   :						Optional[int]      							=					getattr(UpperCamelCase ,       UpperCamelCase  )
																																	# finally set the target attribute
																																	setattr(UpperCamelCase ,       UpperCamelCase ,       self.new  )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

												if submodules:  # if it's an attribute of a submodule like "os.path.join"
																			try:
																										__UpperCAmelCase   :						int      							=					getattr(import_module(""".""".join(UpperCamelCase  )  ) ,       UpperCamelCase  )
																			except (AttributeError, ModuleNotFoundError):
																										return
																			# We iterate over all the globals in self.obj in case we find "os.path.join"
																			for attr in self.obj.__dir__():
																										# We don't check for the name of the global, but rather if its value *is* "os.path.join".
																										# This allows to patch renamed attributes like "from os.path import join as pjoin".
																										if getattr(self.obj ,       UpperCamelCase  ) is attr_value:
																																	__UpperCAmelCase   :						Union[str, Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																																	setattr(self.obj ,       UpperCamelCase ,       self.new  )
												elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
																			__UpperCAmelCase   :						str      							=					globals()["""__builtins__"""][target_attr]
																			setattr(self.obj ,       UpperCamelCase ,       self.new  )
												else:
																			raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''  )






					def __exit__(       self    : str ,       *UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												for attr in list(self.original  ):
																			setattr(self.obj ,       UpperCamelCase ,       self.original.pop(UpperCamelCase  )  )






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												self.__enter__()
												self._active_patches.append(self  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												try:
																			self._active_patches.remove(self  )
												except ValueError:
																			# If the patch hasn't been started this will fail
																			return None

												return self.__exit__()
