


"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase    :     List[str]   						=		logging.get_logger(__name__)

UpperCAmelCase    :     Dict   						=		'▁'

UpperCAmelCase    :     Union[str, Any]   						=		{'vocab_file': 'sentencepiece.bpe.model'}

UpperCAmelCase    :     Optional[int]   						=		{
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

UpperCAmelCase    :     Dict   						=		{
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					__a	       =							VOCAB_FILES_NAMES
					__a	       =							PRETRAINED_VOCAB_FILES_MAP
					__a	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__a	       =							["""input_ids""", """attention_mask"""]






					def __init__(       self    : Union[str, Any] ,       UpperCamelCase    : Dict ,       UpperCamelCase    : List[Any]="<s>" ,       UpperCamelCase    : Any="</s>" ,       UpperCamelCase    : Dict="</s>" ,       UpperCamelCase    : Union[str, Any]="<s>" ,       UpperCamelCase    : int="<unk>" ,       UpperCamelCase    : int="<pad>" ,       UpperCamelCase    : List[Any]="<mask>" ,       UpperCamelCase    : Optional[Dict[str, Any]] = None ,       **UpperCamelCase    : str ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else mask_token

												__UpperCAmelCase   :						List[str]      							=					{} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    bos_token=UpperCamelCase ,       eos_token=UpperCamelCase ,       unk_token=UpperCamelCase ,       sep_token=UpperCamelCase ,       cls_token=UpperCamelCase ,       pad_token=UpperCamelCase ,       mask_token=UpperCamelCase ,       sp_model_kwargs=self.sp_model_kwargs ,       **UpperCamelCase ,       )

												__UpperCAmelCase   :						Dict      							=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
												self.sp_model.Load(str(UpperCamelCase  )  )
												__UpperCAmelCase   :						str      							=					vocab_file

												# Original fairseq vocab and spm vocab must be "aligned":
												# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
												# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
												# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
												# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

												# Mimic fairseq token-to-id alignment for the first 4 token
												__UpperCAmelCase   :						Tuple      							=					{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

												# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
												__UpperCAmelCase   :						Optional[int]      							=					1

												__UpperCAmelCase   :						Any      							=					len(self.sp_model  ) + self.fairseq_offset
												__UpperCAmelCase   :						str      							=					{v: k for k, v in self.fairseq_tokens_to_ids.items()}






					def __getstate__(       self    : int  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					self.__dict__.copy()
												__UpperCAmelCase   :						Union[str, Any]      							=					None
												__UpperCAmelCase   :						Tuple      							=					self.sp_model.serialized_model_proto()
												return state






					def __setstate__(       self    : Union[str, Any] ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					d

												# for backward compatibility
												if not hasattr(self ,       """sp_model_kwargs"""  ):
																			__UpperCAmelCase   :						Tuple      							=					{}

												__UpperCAmelCase   :						str      							=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
												self.sp_model.LoadFromSerializedProto(self.sp_model_proto  )






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None  ):
												'''simple docstring'''






												if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												__UpperCAmelCase   :						str      							=					[self.cls_token_id]
												__UpperCAmelCase   :						Optional[int]      							=					[self.sep_token_id]
												return cls + token_ids_a + sep + sep + token_ids_a + sep






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None ,       UpperCamelCase    : bool = False  ):
												'''simple docstring'''






												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=UpperCamelCase ,       token_ids_a=UpperCamelCase ,       already_has_special_tokens=UpperCamelCase  )

												if token_ids_a is None:
																			return [1] + ([0] * len(UpperCamelCase  )) + [1]
												return [1] + ([0] * len(UpperCamelCase  )) + [1, 1] + ([0] * len(UpperCamelCase  )) + [1]






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					[self.sep_token_id]
												__UpperCAmelCase   :						Optional[int]      							=					[self.cls_token_id]

												if token_ids_a is None:
																			return len(cls + token_ids_a + sep  ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]






					@property
					def lowerCamelCase__							(       self    : List[str]  ):
												'''simple docstring'''






												return len(self.sp_model  ) + self.fairseq_offset + 1  # Add the <mask> token






					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					{self.convert_ids_to_tokens(UpperCamelCase  ): i for i in range(self.vocab_size  )}
												vocab.update(self.added_tokens_encoder  )
												return vocab






					def lowerCamelCase__							(       self    : Any ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												return self.sp_model.encode(UpperCamelCase ,       out_type=UpperCamelCase  )






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : Tuple  ):
												'''simple docstring'''






												if token in self.fairseq_tokens_to_ids:
																			return self.fairseq_tokens_to_ids[token]
												__UpperCAmelCase   :						Any      							=					self.sp_model.PieceToId(UpperCamelCase  )

												# Need to return unknown token if the SP model returned 0
												return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : Union[str, Any]  ):
												'''simple docstring'''






												if index in self.fairseq_ids_to_tokens:
																			return self.fairseq_ids_to_tokens[index]
												return self.sp_model.IdToPiece(index - self.fairseq_offset  )






					def lowerCamelCase__							(       self    : Any ,       UpperCamelCase    : Tuple  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[Any]      							=					"""""".join(UpperCamelCase  ).replace(UpperCamelCase ,       """ """  ).strip()
												return out_string






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : str ,       UpperCamelCase    : Optional[str] = None  ):
												'''simple docstring'''






												if not os.path.isdir(UpperCamelCase  ):
																			logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
																			return
												__UpperCAmelCase   :						Union[str, Any]      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

												if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
																			copyfile(self.vocab_file ,       UpperCamelCase  )
												elif not os.path.isfile(self.vocab_file  ):
																			with open(UpperCamelCase ,       """wb"""  ) as fi:
																										__UpperCAmelCase   :						List[str]      							=					self.sp_model.serialized_model_proto()
																										fi.write(UpperCamelCase  )

												return (out_vocab_file,)




"""simple docstring"""






import numpy as np

UpperCAmelCase    :     Optional[Any]   						=		[
    ['a', 'b', 'c', 'd', 'e'],
    ['f', 'g', 'h', 'i', 'k'],
    ['l', 'm', 'n', 'o', 'p'],
    ['q', 'r', 's', 't', 'u'],
    ['v', 'w', 'x', 'y', 'z'],
]





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					np.array(UpperCamelCase  )






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												__UpperCAmelCase							,__UpperCAmelCase   :						Optional[Any]      							=					np.where(letter == self.SQUARE  )
												__UpperCAmelCase   :						Optional[Any]      							=					np.concatenate([indexa + 1, indexa + 1]  )
												return indexes






					def lowerCamelCase__							(       self    : Any ,       UpperCamelCase    : int ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					self.SQUARE[indexa - 1, indexa - 1]
												return letter






					def lowerCamelCase__							(       self    : Optional[int] ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					message.lower()
												__UpperCAmelCase   :						List[Any]      							=					message.replace(""" """ ,       """"""  )
												__UpperCAmelCase   :						List[Any]      							=					message.replace("""j""" ,       """i"""  )

												__UpperCAmelCase   :						Optional[int]      							=					np.empty((2, len(UpperCamelCase  ))  )
												for letter_index in range(len(UpperCamelCase  )  ):
																			__UpperCAmelCase   :						List[Any]      							=					self.letter_to_numbers(message[letter_index]  )

																			__UpperCAmelCase   :						str      							=					numbers[0]
																			__UpperCAmelCase   :						int      							=					numbers[1]

												__UpperCAmelCase   :						Union[str, Any]      							=					first_step.reshape(2 * len(UpperCamelCase  )  )
												__UpperCAmelCase   :						Optional[Any]      							=					""""""
												for numbers_index in range(len(UpperCamelCase  )  ):
																			__UpperCAmelCase   :						Any      							=					int(second_step[numbers_index * 2]  )
																			__UpperCAmelCase   :						Any      							=					int(second_step[(numbers_index * 2) + 1]  )
																			__UpperCAmelCase   :						str      							=					self.numbers_to_letter(UpperCamelCase ,       UpperCamelCase  )
																			__UpperCAmelCase   :						Optional[Any]      							=					encoded_message + letter

												return encoded_message






					def lowerCamelCase__							(       self    : Optional[int] ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					message.lower()
												message.replace(""" """ ,       """"""  )
												__UpperCAmelCase   :						int      							=					np.empty(2 * len(UpperCamelCase  )  )
												for letter_index in range(len(UpperCamelCase  )  ):
																			__UpperCAmelCase   :						Any      							=					self.letter_to_numbers(message[letter_index]  )
																			__UpperCAmelCase   :						Any      							=					numbers[0]
																			__UpperCAmelCase   :						Dict      							=					numbers[1]

												__UpperCAmelCase   :						str      							=					first_step.reshape((2, len(UpperCamelCase  ))  )
												__UpperCAmelCase   :						Union[str, Any]      							=					""""""
												for numbers_index in range(len(UpperCamelCase  )  ):
																			__UpperCAmelCase   :						Optional[int]      							=					int(second_step[0, numbers_index]  )
																			__UpperCAmelCase   :						Tuple      							=					int(second_step[1, numbers_index]  )
																			__UpperCAmelCase   :						Tuple      							=					self.numbers_to_letter(UpperCamelCase ,       UpperCamelCase  )
																			__UpperCAmelCase   :						Any      							=					decoded_message + letter

												return decoded_message
