



"""simple docstring"""

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__    :  Union[str, Any]							    =	logging.get_logger(__name__)

lowercase__    :  Tuple							    =	'''▁'''

lowercase__    :  Any							    =	{'''vocab_file''': '''sentencepiece.bpe.model'''}

lowercase__    :  Union[str, Any]							    =	{
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    }
}

lowercase__    :  Optional[int]							    =	{
    '''facebook/xglm-564M''': 20_48,
}




class        _UpperCAmelCase       (			lowerCAmelCase__):
 _lowerCAmelCase   :			Tuple							=		VOCAB_FILES_NAMES
 _lowerCAmelCase   :			List[Any]							=		PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase   :			str							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase   :			List[str]							=		["""input_ids""", """attention_mask"""]

 def __init__(     self		:       List[Any]  ,     lowercase_		:       List[Any]  ,     lowercase_		:       Tuple="<s>"  ,     lowercase_		:       int="</s>"  ,     lowercase_		:       str="</s>"  ,     lowercase_		:       int="<s>"  ,     lowercase_		:       Optional[Any]="<unk>"  ,     lowercase_		:       Union[str, Any]="<pad>"  ,     lowercase_		:       Optional[Dict[str, Any]] = None  ,     **lowercase_		:       Optional[int]  ,     ):
     snake_case_ :     Optional[Any]				=	{} if sp_model_kwargs is None else sp_model_kwargs

     # Compatibility with the original tokenizer
     snake_case_ :     Any				=	7
     snake_case_ :     Any				=	[f"<madeupword{i}>" for i in range(self.num_madeup_words				)]

     snake_case_ :     str				=	kwargs.get('''additional_special_tokens'''  ,     []				)
     kwargs["additional_special_tokens"] += [
         word for word in madeup_words if word not in kwargs["additional_special_tokens"]
     ]

     super().__init__(
         bos_token=lowercase_  ,     eos_token=lowercase_  ,     unk_token=lowercase_  ,     sep_token=lowercase_  ,     cls_token=lowercase_  ,     pad_token=lowercase_  ,     sp_model_kwargs=self.sp_model_kwargs  ,     **lowercase_  ,     )

     snake_case_ :     Dict				=	spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(str(lowercase_				)				)
     snake_case_ :     Optional[Any]				=	vocab_file

     # Original fairseq vocab and spm vocab must be "aligned":
     # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
     # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
     # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
     # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

     # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
     snake_case_ :     int				=	1

     # Mimic fairseq token-to-id alignment for the first 4 token
     snake_case_ :     str				=	{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

     snake_case_ :     Tuple				=	len(self.sp_model				)
     snake_case_ :     int				=	{f"<madeupword{i}>": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words				)}
     self.fairseq_tokens_to_ids.update(lowercase_				)

     snake_case_ :     int				=	{v: k for k, v in self.fairseq_tokens_to_ids.items()}

 def __getstate__(     self		:       Tuple				):
     snake_case_ :     Tuple				=	self.__dict__.copy()
     snake_case_ :     Union[str, Any]				=	None
     snake_case_ :     int				=	self.sp_model.serialized_model_proto()
     return state

 def __setstate__(     self		:       Tuple  ,     lowercase_		:       Dict				):
     snake_case_ :     int				=	d

     # for backward compatibility
     if not hasattr(self  ,     '''sp_model_kwargs'''				):
         snake_case_ :     Any				=	{}

     snake_case_ :     List[str]				=	spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.LoadFromSerializedProto(self.sp_model_proto				)

 def 						_snake_case					(     self		:       str  ,     lowercase_		:       List[int]  ,     lowercase_		:       Optional[List[int]] = None				):

     if token_ids_a is None:
         return [self.sep_token_id] + token_ids_a
     snake_case_ :     Optional[Any]				=	[self.sep_token_id]
     return sep + token_ids_a + sep + sep + token_ids_a

 def 						_snake_case					(     self		:       Any  ,     lowercase_		:       List[int]  ,     lowercase_		:       Optional[List[int]] = None  ,     lowercase_		:       bool = False				):

     if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowercase_  ,     token_ids_a=lowercase_  ,     already_has_special_tokens=lowercase_				)

     if token_ids_a is None:
         return [1] + ([0] * len(lowercase_				))
     return [1] + ([0] * len(lowercase_				)) + [1, 1] + ([0] * len(lowercase_				))

 def 						_snake_case					(     self		:       List[str]  ,     lowercase_		:       List[int]  ,     lowercase_		:       Optional[List[int]] = None				):
     snake_case_ :     Any				=	[self.sep_token_id]

     if token_ids_a is None:
         return len(sep + token_ids_a				) * [0]
     return len(sep + token_ids_a + sep + sep + token_ids_a				) * [0]

 @property
 def 						_snake_case					(     self		:       Optional[Any]				):
     return len(self.sp_model				) + self.fairseq_offset + self.num_madeup_words

 def 						_snake_case					(     self		:       Union[str, Any]				):
     snake_case_ :     Tuple				=	{self.convert_ids_to_tokens(lowercase_				): i for i in range(self.vocab_size				)}
     vocab.update(self.added_tokens_encoder				)
     return vocab

 def 						_snake_case					(     self		:       Union[str, Any]  ,     lowercase_		:       str				):
     return self.sp_model.encode(lowercase_  ,     out_type=lowercase_				)

 def 						_snake_case					(     self		:       int  ,     lowercase_		:       int				):
     if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
     snake_case_ :     Dict				=	self.sp_model.PieceToId(lowercase_				)

     # Need to return unknown token if the SP model returned 0
     return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

 def 						_snake_case					(     self		:       List[str]  ,     lowercase_		:       Union[str, Any]				):
     if index in self.fairseq_ids_to_tokens:
         return self.fairseq_ids_to_tokens[index]
     return self.sp_model.IdToPiece(index - self.fairseq_offset				)

 def 						_snake_case					(     self		:       Optional[int]  ,     lowercase_		:       Optional[int]				):
     snake_case_ :     Optional[int]				=	''''''.join(lowercase_				).replace(lowercase_  ,     ''' '''				).strip()
     return out_string






 def 						_snake_case					(     self		:       int  ,     lowercase_		:       str  ,     lowercase_		:       Optional[str] = None				):
     if not os.path.isdir(lowercase_				):
         logger.error(f"Vocabulary path ({save_directory}) should be a directory"				)
         return
     snake_case_ :     str				=	os.path.join(
         lowercase_  ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

     if os.path.abspath(self.vocab_file				) != os.path.abspath(lowercase_				) and os.path.isfile(self.vocab_file				):
         copyfile(self.vocab_file  ,     lowercase_				)
     elif not os.path.isfile(self.vocab_file				):
         with open(lowercase_  ,     '''wb'''				) as fi:
             snake_case_ :     Union[str, Any]				=	self.sp_model.serialized_model_proto()
             fi.write(lowercase_				)

     return (out_vocab_file,)










"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase__    :  List[Any]							    =	{
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Any							    =	['''DistilBertTokenizerFast''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  int							    =	[
         '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''DistilBertForMaskedLM''',
         '''DistilBertForMultipleChoice''',
         '''DistilBertForQuestionAnswering''',
         '''DistilBertForSequenceClassification''',
         '''DistilBertForTokenClassification''',
         '''DistilBertModel''',
         '''DistilBertPreTrainedModel''',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Dict							    =	[
         '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''TFDistilBertForMaskedLM''',
         '''TFDistilBertForMultipleChoice''',
         '''TFDistilBertForQuestionAnswering''',
         '''TFDistilBertForSequenceClassification''',
         '''TFDistilBertForTokenClassification''',
         '''TFDistilBertMainLayer''',
         '''TFDistilBertModel''',
         '''TFDistilBertPreTrainedModel''',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Tuple							    =	[
         '''FlaxDistilBertForMaskedLM''',
         '''FlaxDistilBertForMultipleChoice''',
         '''FlaxDistilBertForQuestionAnswering''',
         '''FlaxDistilBertForSequenceClassification''',
         '''FlaxDistilBertForTokenClassification''',
         '''FlaxDistilBertModel''',
         '''FlaxDistilBertPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_distilbert import (
         DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         DistilBertConfig,
         DistilBertOnnxConfig,
     )
     from .tokenization_distilbert import DistilBertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_distilbert_fast import DistilBertTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_distilbert import (
              DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
              DistilBertModel,
              DistilBertPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_distilbert import (
              TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFDistilBertForMaskedLM,
              TFDistilBertForMultipleChoice,
              TFDistilBertForQuestionAnswering,
              TFDistilBertForSequenceClassification,
              TFDistilBertForTokenClassification,
              TFDistilBertMainLayer,
              TFDistilBertModel,
              TFDistilBertPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_distilbert import (
              FlaxDistilBertForMaskedLM,
              FlaxDistilBertForMultipleChoice,
              FlaxDistilBertForQuestionAnswering,
              FlaxDistilBertForSequenceClassification,
              FlaxDistilBertForTokenClassification,
              FlaxDistilBertModel,
              FlaxDistilBertPreTrainedModel,
          )

else:
     import sys

     lowercase__    :  Union[str, Any]							    =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





