




"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase__     						=						logging.get_logger(__name__)

lowercase__     						=						"""▁"""

lowercase__     						=						{"""vocab_file""": """sentencepiece.bpe.model"""}

lowercase__     						=						{
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

lowercase__     						=						{
    """xlm-roberta-base""": 512,
    """xlm-roberta-large""": 512,
    """xlm-roberta-large-finetuned-conll02-dutch""": 512,
    """xlm-roberta-large-finetuned-conll02-spanish""": 512,
    """xlm-roberta-large-finetuned-conll03-english""": 512,
    """xlm-roberta-large-finetuned-conll03-german""": 512,
}






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			VOCAB_FILES_NAMES
			lowerCamelCase__      =			PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__      =			["""input_ids""", """attention_mask"""]
			def __init__(  self  ,      lowercase  ,      lowercase="<s>"  ,      lowercase="</s>"  ,      lowercase="</s>"  ,      lowercase="<s>"  ,      lowercase="<unk>"  ,      lowercase="<pad>"  ,      lowercase="<mask>"  ,      lowercase = None  ,      **lowercase  ,      ):
							# Mask token behave like a normal word, i.e. include the space before it
							_lowerCamelCase     :			List[Any]    =    AddedToken(lowercase  ,      lstrip=lowercase  ,      rstrip=lowercase  ) if isinstance(lowercase  ,      lowercase  ) else mask_token

							_lowerCamelCase     :			Dict    =    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=lowercase  ,      eos_token=lowercase  ,      unk_token=lowercase  ,      sep_token=lowercase  ,      cls_token=lowercase  ,      pad_token=lowercase  ,      mask_token=lowercase  ,      sp_model_kwargs=self.sp_model_kwargs  ,      **lowercase  ,      )

							_lowerCamelCase     :			str    =    spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(str(lowercase  )  )
							_lowerCamelCase     :			int    =    vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							_lowerCamelCase     :			Optional[int]    =    {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_lowerCamelCase     :			List[Any]    =    1

							_lowerCamelCase     :			Dict    =    len(self.sp_model  ) + self.fairseq_offset
							_lowerCamelCase     :			Tuple    =    {v: k for k, v in self.fairseq_tokens_to_ids.items()}
			def __getstate__(  self  ):
							_lowerCamelCase     :			int    =    self.__dict__.copy()
							_lowerCamelCase     :			str    =    None
							_lowerCamelCase     :			Optional[Any]    =    self.sp_model.serialized_model_proto()
							return state
			def __setstate__(  self  ,      lowercase  ):
							_lowerCamelCase     :			Optional[Any]    =    d

							# for backward compatibility
							if not hasattr(self  ,      'sp_model_kwargs'  ):
											_lowerCamelCase     :			Optional[Any]    =    {}

							_lowerCamelCase     :			int    =    spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto  )
			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):

							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_lowerCamelCase     :			Optional[int]    =    [self.cls_token_id]
							_lowerCamelCase     :			Union[str, Any]    =    [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep
			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ,      lowercase = False  ):

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowercase  ,      token_ids_a=lowercase  ,      already_has_special_tokens=lowercase  )

							if token_ids_a is None:
											return [1] + ([0] * len(lowercase  )) + [1]
							return [1] + ([0] * len(lowercase  )) + [1, 1] + ([0] * len(lowercase  )) + [1]
			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							_lowerCamelCase     :			Dict    =    [self.sep_token_id]
							_lowerCamelCase     :			Dict    =    [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep  ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]
			@property
			def 					A_    (  self  ):
							return len(self.sp_model  ) + self.fairseq_offset + 1  # Add the <mask> token
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    {self.convert_ids_to_tokens(lowercase  ): i for i in range(self.vocab_size  )}
							vocab.update(self.added_tokens_encoder  )
							return vocab
			def 					A_    (  self  ,      lowercase  ):
							return self.sp_model.encode(lowercase  ,      out_type=lowercase  )
			def 					A_    (  self  ,      lowercase  ):
							if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
							_lowerCamelCase     :			List[str]    =    self.sp_model.PieceToId(lowercase  )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
			def 					A_    (  self  ,      lowercase  ):
							if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset  )
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Union[str, Any]    =    ''.join(lowercase  ).replace(lowercase  ,      ' '  ).strip()
							return out_string






			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							if not os.path.isdir(lowercase  ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
											return
							_lowerCamelCase     :			Any    =    os.path.join(
							    lowercase  ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

							if os.path.abspath(self.vocab_file  ) != os.path.abspath(lowercase  ) and os.path.isfile(self.vocab_file  ):
											copyfile(self.vocab_file  ,      lowercase  )
							elif not os.path.isfile(self.vocab_file  ):
											with open(lowercase  ,      'wb'  ) as fi:
															_lowerCamelCase     :			Optional[Any]    =    self.sp_model.serialized_model_proto()
															fi.write(lowercase  )

							return (out_vocab_file,)





"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase__     						=						{"""configuration_ibert""": ["""IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """IBertConfig""", """IBertOnnxConfig"""]}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowercase__     						=						[
			    """IBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
			    """IBertForMaskedLM""",
			    """IBertForMultipleChoice""",
			    """IBertForQuestionAnswering""",
			    """IBertForSequenceClassification""",
			    """IBertForTokenClassification""",
			    """IBertModel""",
			    """IBertPreTrainedModel""",
			]

if TYPE_CHECKING:
			from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_ibert import (
						    IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    IBertForMaskedLM,
						    IBertForMultipleChoice,
						    IBertForQuestionAnswering,
						    IBertForSequenceClassification,
						    IBertForTokenClassification,
						    IBertModel,
						    IBertPreTrainedModel,
						)

else:
			import sys

			lowercase__     						=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)