


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case				:    Dict       				=     logging.get_logger(__name__)

snake_case				:    Tuple       				=     '''▁'''

snake_case				:    Any       				=     {'''vocab_file''': '''sentencepiece.bpe.model'''}

snake_case				:    Tuple       				=     {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

snake_case				:    int       				=     {
    '''xlm-roberta-base''': 5_12,
    '''xlm-roberta-large''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_12,
}







class 		_snake_case  (      _snake_case							):
 SCREAMING_SNAKE_CASE__      =       VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE__      =       ['input_ids', 'attention_mask']




 def __init__(		self					,  _lowerCamelCase					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="<unk>"					,  _lowerCamelCase="<pad>"					,  _lowerCamelCase="<mask>"					,  _lowerCamelCase = None					,  **_lowerCamelCase					,  ):
       # Mask token behave like a normal word, i.e. include the space before it
       a						:Optional[int] 							=       AddedToken(_lowerCamelCase					,  lstrip=_lowerCamelCase					,  rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase					,  _lowerCamelCase    ) else mask_token

       a						:int 							=       {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           bos_token=_lowerCamelCase					,  eos_token=_lowerCamelCase					,  unk_token=_lowerCamelCase					,  sep_token=_lowerCamelCase					,  cls_token=_lowerCamelCase					,  pad_token=_lowerCamelCase					,  mask_token=_lowerCamelCase					,  sp_model_kwargs=self.sp_model_kwargs					,  **_lowerCamelCase					,  )

       a						:Optional[int] 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(str(_lowerCamelCase    )    )
       a						:str 							=       vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # Mimic fairseq token-to-id alignment for the first 4 token
       a						:Tuple 							=       {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       a						:List[str] 							=       1

       a						:Dict 							=       len(self.sp_model    ) + self.fairseq_offset
       a						:List[Any] 							=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}




 def __getstate__(		self    ):
       a						:List[str] 							=       self.__dict__.copy()
       a						:Optional[int] 							=       None
       a						:int 							=       self.sp_model.serialized_model_proto()
       return state




 def __setstate__(		self					,  _lowerCamelCase    ):
       a						:Union[str, Any] 							=       d

       # for backward compatibility
       if not hasattr(self					,  '''sp_model_kwargs'''    ):
             a						:Union[str, Any] 							=       {}

       a						:Optional[int] 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):

       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       a						:List[Any] 							=       [self.cls_token_id]
       a						:Dict 							=       [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None					,  _lowerCamelCase = False    ):

       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_lowerCamelCase					,  token_ids_a=_lowerCamelCase					,  already_has_special_tokens=_lowerCamelCase    )

       if token_ids_a is None:
             return [1] + ([0] * len(_lowerCamelCase    )) + [1]
       return [1] + ([0] * len(_lowerCamelCase    )) + [1, 1] + ([0] * len(_lowerCamelCase    )) + [1]




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       a						:int 							=       [self.sep_token_id]
       a						:int 							=       [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       return len(self.sp_model    ) + self.fairseq_offset + 1  # Add the <mask> token




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Any 							=       {self.convert_ids_to_tokens(_lowerCamelCase    ): i for i in range(self.vocab_size    )}
       vocab.update(self.added_tokens_encoder    )
       return vocab




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       return self.sp_model.encode(_lowerCamelCase					,  out_type=_lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
       a						:Optional[Any] 							=       self.sp_model.PieceToId(_lowerCamelCase    )

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       a						:Tuple 							=       ''''''.join(_lowerCamelCase    ).replace(_lowerCamelCase					,  ''' '''    ).strip()
       return out_string




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       if not os.path.isdir(_lowerCamelCase    ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
             return
       a						:int 							=       os.path.join(
           _lowerCamelCase					,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(_lowerCamelCase    ) and os.path.isfile(self.vocab_file    ):
             copyfile(self.vocab_file					,  _lowerCamelCase    )
       elif not os.path.isfile(self.vocab_file    ):
             with open(_lowerCamelCase					,  '''wb'''    ) as fi:
                   a						:List[Any] 							=       self.sp_model.serialized_model_proto()
                   fi.write(_lowerCamelCase    )

       return (out_vocab_file,)









from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


snake_case				:    List[str]       				=     logging.get_logger(__name__)

snake_case				:    int       				=     {
    '''facebook/blenderbot_small-90M''': '''https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/config.json''',
    # See all BlenderbotSmall models at https://huggingface.co/models?filter=blenderbot_small
}







class 		_snake_case  (      _snake_case							):
 SCREAMING_SNAKE_CASE__      =       'blenderbot-small'
 SCREAMING_SNAKE_CASE__      =       ['past_key_values']
 SCREAMING_SNAKE_CASE__      =       {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}




 def __init__(		self					,  _lowerCamelCase=5_0265					,  _lowerCamelCase=512					,  _lowerCamelCase=8					,  _lowerCamelCase=2048					,  _lowerCamelCase=16					,  _lowerCamelCase=8					,  _lowerCamelCase=2048					,  _lowerCamelCase=16					,  _lowerCamelCase=0.0					,  _lowerCamelCase=0.0					,  _lowerCamelCase=True					,  _lowerCamelCase=True					,  _lowerCamelCase="gelu"					,  _lowerCamelCase=512					,  _lowerCamelCase=0.1					,  _lowerCamelCase=0.0					,  _lowerCamelCase=0.0					,  _lowerCamelCase=0.02					,  _lowerCamelCase=1					,  _lowerCamelCase=False					,  _lowerCamelCase=0					,  _lowerCamelCase=1					,  _lowerCamelCase=2					,  _lowerCamelCase=2					,  **_lowerCamelCase					,  ):
       a						:Dict 							=       vocab_size
       a						:Optional[Any] 							=       max_position_embeddings
       a						:str 							=       d_model
       a						:Any 							=       encoder_ffn_dim
       a						:Optional[int] 							=       encoder_layers
       a						:List[str] 							=       encoder_attention_heads
       a						:List[str] 							=       decoder_ffn_dim
       a						:Optional[int] 							=       decoder_layers
       a						:str 							=       decoder_attention_heads
       a						:List[str] 							=       dropout
       a						:Optional[int] 							=       attention_dropout
       a						:Dict 							=       activation_dropout
       a						:List[str] 							=       activation_function
       a						:List[Any] 							=       init_std
       a						:Optional[int] 							=       encoder_layerdrop
       a						:Tuple 							=       decoder_layerdrop
       a						:List[str] 							=       use_cache
       a						:int 							=       encoder_layers
       a						:Union[str, Any] 							=       scale_embedding  # scale factor will be sqrt(d_model) if True

       super().__init__(
           pad_token_id=_lowerCamelCase					,  bos_token_id=_lowerCamelCase					,  eos_token_id=_lowerCamelCase					,  is_encoder_decoder=_lowerCamelCase					,  decoder_start_token_id=_lowerCamelCase					,  forced_eos_token_id=_lowerCamelCase					,  **_lowerCamelCase					,  )







class 		_snake_case  (      _snake_case							):




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       if self.task in ["default", "seq2seq-lm"]:
             a						:Optional[Any] 							=       OrderedDict(
                 [
                     ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                 ]    )

             if self.use_past:
                   a						:Union[str, Any] 							=       {0: '''batch'''}
                   a						:Tuple 							=       {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
             else:
                   a						:Optional[int] 							=       {0: '''batch''', 1: '''decoder_sequence'''}
                   a						:str 							=       {0: '''batch''', 1: '''decoder_sequence'''}

             if self.use_past:
                   self.fill_with_past_key_values_(_lowerCamelCase					,  direction='''inputs'''    )
       elif self.task == "causal-lm":
             # TODO: figure this case out.
             a						:Optional[int] 							=       OrderedDict(
                 [
                     ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                 ]    )
             if self.use_past:
                   a							,				a						:str 							=       self.num_layers
                   for i in range(_lowerCamelCase    ):
                         a						:List[Any] 							=       {0: '''batch''', 2: '''past_sequence + sequence'''}
                         a						:List[str] 							=       {0: '''batch''', 2: '''past_sequence + sequence'''}
       else:
             a						:Optional[int] 							=       OrderedDict(
                 [
                     ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                     ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
                 ]    )

       return common_inputs




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       if self.task in ["default", "seq2seq-lm"]:
             a						:List[Any] 							=       super().outputs
       else:
             a						:Union[str, Any] 							=       super(_lowerCamelCase					,  self    ).outputs
             if self.use_past:
                   a							,				a						:int 							=       self.num_layers
                   for i in range(_lowerCamelCase    ):
                         a						:int 							=       {0: '''batch''', 2: '''past_sequence + sequence'''}
                         a						:Optional[Any] 							=       {0: '''batch''', 2: '''past_sequence + sequence'''}
       return common_outputs




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = -1					,  _lowerCamelCase = -1					,  _lowerCamelCase = False					,  _lowerCamelCase = None					,  ):
       a						:Tuple 							=       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    )

       # Generate decoder inputs
       a						:Dict 							=       seq_length if not self.use_past else 1
       a						:Dict 							=       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    )
       a						:List[Any] 							=       {F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
       a						:List[str] 							=       dict(**_lowerCamelCase					,  **_lowerCamelCase    )

       if self.use_past:
             if not is_torch_available():
                   raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''    )
             else:
                   import torch
             a							,				a						:Optional[Any] 							=       common_inputs['''input_ids'''].shape
             a						:Tuple 							=       common_inputs['''decoder_input_ids'''].shape[1]
             a							,				a						:List[Any] 							=       self.num_attention_heads
             a						:List[Any] 							=       (
                 batch,
                 num_encoder_attention_heads,
                 encoder_seq_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )
             a						:int 							=       decoder_seq_length + 3
             a						:Union[str, Any] 							=       (
                 batch,
                 num_decoder_attention_heads,
                 decoder_past_length,
                 self._config.hidden_size // num_decoder_attention_heads,
             )

             a						:Union[str, Any] 							=       torch.cat(
                 [common_inputs['''decoder_attention_mask'''], torch.ones(_lowerCamelCase					,  _lowerCamelCase    )]					,  dim=1    )

             a						:List[Any] 							=       []
             # If the number of encoder and decoder layers are present in the model configuration, both are considered
             a							,				a						:Optional[int] 							=       self.num_layers
             a						:str 							=       min(_lowerCamelCase					,  _lowerCamelCase    )
             a						:str 							=       max(_lowerCamelCase					,  _lowerCamelCase    ) - min_num_layers
             a						:Tuple 							=       '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

             for _ in range(_lowerCamelCase    ):
                   common_inputs["past_key_values"].append(
                       (
                           torch.zeros(_lowerCamelCase    ),
                           torch.zeros(_lowerCamelCase    ),
                           torch.zeros(_lowerCamelCase    ),
                           torch.zeros(_lowerCamelCase    ),
                       )    )
             # TODO: test this.
             a						:int 							=       encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
             for _ in range(_lowerCamelCase					,  _lowerCamelCase    ):
                   common_inputs["past_key_values"].append((torch.zeros(_lowerCamelCase    ), torch.zeros(_lowerCamelCase    ))    )
       return common_inputs




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = -1					,  _lowerCamelCase = -1					,  _lowerCamelCase = False					,  _lowerCamelCase = None					,  ):
       a						:Tuple 							=       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    )

       if self.use_past:
             if not is_torch_available():
                   raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''    )
             else:
                   import torch
             a							,				a						:Dict 							=       common_inputs['''input_ids'''].shape
             # Not using the same length for past_key_values
             a						:Optional[int] 							=       seqlen + 2
             a							,				a						:Union[str, Any] 							=       self.num_layers
             a							,				a						:Optional[Any] 							=       self.num_attention_heads
             a						:str 							=       (
                 batch,
                 num_encoder_attention_heads,
                 past_key_values_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )

             a						:Tuple 							=       common_inputs['''attention_mask'''].dtype
             a						:Any 							=       torch.cat(
                 [common_inputs['''attention_mask'''], torch.ones(_lowerCamelCase					,  _lowerCamelCase					,  dtype=_lowerCamelCase    )]					,  dim=1    )
             a						:Any 							=       [
                 (torch.zeros(_lowerCamelCase    ), torch.zeros(_lowerCamelCase    )) for _ in range(_lowerCamelCase    )
             ]
       return common_inputs




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = -1					,  _lowerCamelCase = -1					,  _lowerCamelCase = False					,  _lowerCamelCase = None					,  ):
       # Copied from OnnxConfig.generate_dummy_inputs
       # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
       # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
       a						:Optional[Any] 							=       compute_effective_axis_dimension(
           _lowerCamelCase					,  fixed_dimension=OnnxConfig.default_fixed_batch					,  num_token_to_add=0    )

       # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
       a						:Optional[int] 							=       tokenizer.num_special_tokens_to_add(_lowerCamelCase    )
       a						:Tuple 							=       compute_effective_axis_dimension(
           _lowerCamelCase					,  fixed_dimension=OnnxConfig.default_fixed_sequence					,  num_token_to_add=_lowerCamelCase    )

       # Generate dummy inputs according to compute batch and sequence
       a						:List[str] 							=       [''' '''.join([tokenizer.unk_token]    ) * seq_length] * batch_size
       a						:Dict 							=       dict(tokenizer(_lowerCamelCase					,  return_tensors=_lowerCamelCase    )    )
       return common_inputs




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = -1					,  _lowerCamelCase = -1					,  _lowerCamelCase = False					,  _lowerCamelCase = None					,  ):
       if self.task in ["default", "seq2seq-lm"]:
             a						:Tuple 							=       self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                 _lowerCamelCase					,  batch_size=_lowerCamelCase					,  seq_length=_lowerCamelCase					,  is_pair=_lowerCamelCase					,  framework=_lowerCamelCase    )

       elif self.task == "causal-lm":
             a						:Dict 							=       self._generate_dummy_inputs_for_causal_lm(
                 _lowerCamelCase					,  batch_size=_lowerCamelCase					,  seq_length=_lowerCamelCase					,  is_pair=_lowerCamelCase					,  framework=_lowerCamelCase    )
       else:
             a						:Dict 							=       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 _lowerCamelCase					,  batch_size=_lowerCamelCase					,  seq_length=_lowerCamelCase					,  is_pair=_lowerCamelCase					,  framework=_lowerCamelCase    )

       return common_inputs




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       if self.task in ["default", "seq2seq-lm"]:
             a						:Optional[int] 							=       super()._flatten_past_key_values_(_lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    )
       else:
             a						:Any 							=       super(_lowerCamelCase					,  self    )._flatten_past_key_values_(
                 _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    )





