





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


snake_case								=							logging.get_logger(__name__)

snake_case								=							"""▁"""

snake_case								=							{"""vocab_file""": """sentencepiece.bpe.model"""}

snake_case								=							{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

snake_case								=							{
    """facebook/mbart-large-en-ro""": 1_0_2_4,
    """facebook/mbart-large-cc25""": 1_0_2_4,
}

# fmt: off
snake_case								=							["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]





class 					A_					(   A__						):







       """simple docstring"""





       SCREAMING_SNAKE_CASE_   :			Optional[int]  			=	VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE_   :			Tuple  			=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE_   :			Union[str, Any]  			=	PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE_   :			List[str]  			=	['''input_ids''', '''attention_mask''']

       SCREAMING_SNAKE_CASE_   :			int  			=	[]
       SCREAMING_SNAKE_CASE_   :			Union[str, Any]  			=	[]







       def __init__(     self						:						str    ,__A						:						Tuple    ,__A						:						Union[str, Any]="<s>"    ,__A						:						Optional[Any]="</s>"    ,__A						:						Optional[Any]="</s>"    ,__A						:						Union[str, Any]="<s>"    ,__A						:						Optional[int]="<unk>"    ,__A						:						Union[str, Any]="<pad>"    ,__A						:						int="<mask>"    ,__A						:						int=None    ,__A						:						Dict=None    ,__A						:						Tuple=None    ,__A						:						List[str] = None    ,__A						:						Dict=None    ,**__A						:						Any    ,)    ->       Tuple:
         # Mask token behave like a normal word, i.e. include the space before it
         _lowercase						    =			AddedToken(a__    ,lstrip=a__    ,rstrip=a__	) if isinstance(a__    ,a__	) else mask_token

         _lowercase						    =			{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=a__    ,eos_token=a__    ,unk_token=a__    ,sep_token=a__    ,cls_token=a__    ,pad_token=a__    ,mask_token=a__    ,tokenizer_file=a__    ,src_lang=a__    ,tgt_lang=a__    ,additional_special_tokens=a__    ,sp_model_kwargs=self.sp_model_kwargs    ,**a__    ,)

         _lowercase						    =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
         self.sp_model.Load(str(a__	)	)
         _lowercase						    =			vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # Mimic fairseq token-to-id alignment for the first 4 token
         _lowercase						    =			{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         _lowercase						    =			1

         _lowercase						    =			len(self.sp_model	)
         _lowercase						    =			{
             code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(a__	)
         }
         _lowercase						    =			{v: k for k, v in self.lang_code_to_id.items()}
         _lowercase						    =			len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset

         self.fairseq_tokens_to_ids.update(self.lang_code_to_id	)
         _lowercase						    =			{v: k for k, v in self.fairseq_tokens_to_ids.items()}
         _lowercase						    =			list(self.lang_code_to_id.keys()	)

         if additional_special_tokens is not None:
           # Only add those special tokens if they are not already there.
           self._additional_special_tokens.extend(
               [t for t in additional_special_tokens if t not in self._additional_special_tokens]	)

         _lowercase						    =			src_lang if src_lang is not None else 'en_XX'
         _lowercase						    =			self.lang_code_to_id[self._src_lang]
         _lowercase						    =			tgt_lang
         self.set_src_lang_special_tokens(self._src_lang	)







       def __getstate__(     self						:						Union[str, Any]	)    ->       Union[str, Any]:
         _lowercase						    =			self.__dict__.copy()
         _lowercase						    =			None
         _lowercase						    =			self.sp_model.serialized_model_proto()
         return state







       def __setstate__(     self						:						str    ,__A						:						Optional[Any]	)    ->       Optional[int]:
         _lowercase						    =			d

         # for backward compatibility
         if not hasattr(self    ,'sp_model_kwargs'	):
           _lowercase						    =			{}

         _lowercase						    =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto	)







       @property
       def  __UpperCAmelCase			(     self						:						Any	)    ->       Dict:
         return len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset + 1  # Plus 1 for the mask token







       @property
       def  __UpperCAmelCase			(     self						:						int	)    ->       Dict:
         return self._src_lang







       @src_lang.setter
       def  __UpperCAmelCase			(     self						:						Optional[Any]    ,__A						:						Union[str, Any]	)    ->       int:
         _lowercase						    =			new_src_lang
         self.set_src_lang_special_tokens(self._src_lang	)







       def  __UpperCAmelCase			(     self						:						List[Any]    ,__A						:						Dict    ,__A						:						Optional[int] = None    ,__A						:						str = False	)    ->       Tuple:

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=a__    ,token_ids_a=a__    ,already_has_special_tokens=a__	)

         _lowercase						    =			[1] * len(self.prefix_tokens	)
         _lowercase						    =			[1] * len(self.suffix_tokens	)
         if token_ids_a is None:
           return prefix_ones + ([0] * len(a__	)) + suffix_ones
         return prefix_ones + ([0] * len(a__	)) + ([0] * len(a__	)) + suffix_ones







       def  __UpperCAmelCase			(     self						:						List[Any]    ,__A						:						Tuple    ,__A						:						List[str] = None	)    ->       Union[str, Any]:
         if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







       def  __UpperCAmelCase			(     self						:						Optional[Any]    ,__A						:						int    ,__A						:						Any = None	)    ->       List[Any]:
         _lowercase						    =			[self.sep_token_id]
         _lowercase						    =			[self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep	) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]







       def  __UpperCAmelCase			(     self						:						str    ,__A						:						Optional[Any]    ,__A						:						List[str]    ,__A						:						Optional[int]    ,__A						:						Any    ,**__A						:						Union[str, Any]	)    ->       Tuple:
         if src_lang is None or tgt_lang is None:
           raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
         _lowercase						    =			src_lang
         _lowercase						    =			self(a__    ,add_special_tokens=a__    ,return_tensors=a__    ,**a__	)
         _lowercase						    =			self.convert_tokens_to_ids(a__	)
         _lowercase						    =			tgt_lang_id
         return inputs







       def  __UpperCAmelCase			(     self						:						List[str]	)    ->       str:
         _lowercase						    =			{self.convert_ids_to_tokens(a__	): i for i in range(self.vocab_size	)}
         vocab.update(self.added_tokens_encoder	)
         return vocab







       def  __UpperCAmelCase			(     self						:						List[Any]    ,__A						:						Dict	)    ->       List[str]:
         return self.sp_model.encode(a__    ,out_type=a__	)







       def  __UpperCAmelCase			(     self						:						List[Any]    ,__A						:						Optional[int]	)    ->       Optional[int]:
         if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
         _lowercase						    =			self.sp_model.PieceToId(a__	)

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







       def  __UpperCAmelCase			(     self						:						List[str]    ,__A						:						Union[str, Any]	)    ->       Tuple:
         if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset	)







       def  __UpperCAmelCase			(     self						:						Optional[int]    ,__A						:						int	)    ->       int:
         _lowercase						    =			''.join(a__	).replace(a__    ,' '	).strip()
         return out_string







       def  __UpperCAmelCase			(     self						:						Tuple    ,__A						:						Any    ,__A						:						Any = None	)    ->       List[Any]:
         if not os.path.isdir(a__	):
           logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
           return
         _lowercase						    =			os.path.join(
             a__    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(a__	) and os.path.isfile(self.vocab_file	):
           copyfile(self.vocab_file    ,a__	)
         elif not os.path.isfile(self.vocab_file	):
           with open(a__    ,'wb'	) as fi:
             _lowercase						    =			self.sp_model.serialized_model_proto()
             fi.write(a__	)

         return (out_vocab_file,)







       def  __UpperCAmelCase			(     self						:						Dict    ,__A						:						Dict    ,__A						:						str = "en_XX"    ,__A						:						Optional[Any] = None    ,__A						:						Optional[int] = "ro_RO"    ,**__A						:						List[str]    ,)    ->       Tuple:
         _lowercase						    =			src_lang
         _lowercase						    =			tgt_lang
         return super().prepare_seqaseq_batch(a__    ,a__    ,**a__	)







       def  __UpperCAmelCase			(     self						:						Optional[Any]	)    ->       Optional[int]:
         return self.set_src_lang_special_tokens(self.src_lang	)







       def  __UpperCAmelCase			(     self						:						str	)    ->       Tuple:
         return self.set_tgt_lang_special_tokens(self.tgt_lang	)







       def  __UpperCAmelCase			(     self						:						Dict    ,__A						:						Dict	)    ->       Optional[Any]:
         _lowercase						    =			self.lang_code_to_id[src_lang]
         _lowercase						    =			[]
         _lowercase						    =			[self.eos_token_id, self.cur_lang_code]







       def  __UpperCAmelCase			(     self						:						Dict    ,__A						:						Tuple	)    ->       Optional[Any]:
         _lowercase						    =			self.lang_code_to_id[lang]
         _lowercase						    =			[]
         _lowercase						    =			[self.eos_token_id, self.cur_lang_code]




from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import (
       TFDebertaVaForMaskedLM,
       TFDebertaVaForQuestionAnswering,
       TFDebertaVaForSequenceClassification,
       TFDebertaVaForTokenClassification,
       TFDebertaVaModel,
   )



class      _UpperCAmelCase      :




     def __init__(					self			,    a__			,    a__=1_3			,    a__=7			,    a__=True			,    a__=True			,    a__=True			,    a__=True			,    a__=9_9			,    a__=3_2			,    a__=2			,    a__=4			,    a__=3_7			,    a__="gelu"			,    a__=0.1			,    a__=0.1			,    a__=5_1_2			,    a__=1_6			,    a__=2			,    a__=0.0_2			,    a__=False			,    a__=True			,    a__="None"			,    a__=3			,    a__=4			,    a__=None			,    ):
         A__					=       parent
         A__					=       batch_size
         A__					=       seq_length
         A__					=       is_training
         A__					=       use_input_mask
         A__					=       use_token_type_ids
         A__					=       use_labels
         A__					=       vocab_size
         A__					=       hidden_size
         A__					=       num_hidden_layers
         A__					=       num_attention_heads
         A__					=       intermediate_size
         A__					=       hidden_act
         A__					=       hidden_dropout_prob
         A__					=       attention_probs_dropout_prob
         A__					=       max_position_embeddings
         A__					=       type_vocab_size
         A__					=       type_sequence_label_size
         A__					=       initializer_range
         A__					=       num_labels
         A__					=       num_choices
         A__					=       relative_attention
         A__					=       position_biased_input
         A__					=       pos_att_type
         A__					=       scope




     def 		snake_case_		(					self):
         A__					=       ids_tensor([self.batch_size, self.seq_length]			,    self.vocab_size)

         A__					=       None
         if self.use_input_mask:
             A__					=       random_attention_mask([self.batch_size, self.seq_length])

         A__					=       None
         if self.use_token_type_ids:
             A__					=       ids_tensor([self.batch_size, self.seq_length]			,    self.type_vocab_size)

         A__					=       None
         A__					=       None
         A__					=       None
         if self.use_labels:
             A__					=       ids_tensor([self.batch_size]			,    self.type_sequence_label_size)
             A__					=       ids_tensor([self.batch_size, self.seq_length]			,    self.num_labels)

         A__					=       DebertaVaConfig(
             vocab_size=self.vocab_size			,    hidden_size=self.hidden_size			,    num_hidden_layers=self.num_hidden_layers			,    num_attention_heads=self.num_attention_heads			,    intermediate_size=self.intermediate_size			,    hidden_act=self.hidden_act			,    hidden_dropout_prob=self.hidden_dropout_prob			,    attention_probs_dropout_prob=self.attention_probs_dropout_prob			,    max_position_embeddings=self.max_position_embeddings			,    type_vocab_size=self.type_vocab_size			,    relative_attention=self.relative_attention			,    position_biased_input=self.position_biased_input			,    initializer_range=self.initializer_range			,    return_dict=a__			,    )

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




     def 		snake_case_		(					self			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__):
         A__					=       TFDebertaVaModel(config=a__)
         A__					=       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

         A__					=       [input_ids, input_mask]
         A__					=       model(a__)

         A__					=       model(a__)

         self.parent.assertEqual(result.last_hidden_state.shape			,    (self.batch_size, self.seq_length, self.hidden_size))




     def 		snake_case_		(					self			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__):
         A__					=       TFDebertaVaForMaskedLM(config=a__)
         A__					=       {
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         A__					=       model(a__)
         self.parent.assertEqual(result.logits.shape			,    (self.batch_size, self.seq_length, self.vocab_size))




     def 		snake_case_		(					self			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__):
         A__					=       self.num_labels
         A__					=       TFDebertaVaForSequenceClassification(config=a__)
         A__					=       {
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         A__					=       model(a__)
         self.parent.assertEqual(result.logits.shape			,    (self.batch_size, self.num_labels))




     def 		snake_case_		(					self			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__):
         A__					=       self.num_labels
         A__					=       TFDebertaVaForTokenClassification(config=a__)
         A__					=       {
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         A__					=       model(a__)
         self.parent.assertEqual(result.logits.shape			,    (self.batch_size, self.seq_length, self.num_labels))




     def 		snake_case_		(					self			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__			,    a__):
         A__					=       TFDebertaVaForQuestionAnswering(config=a__)
         A__					=       {
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         A__					=       model(a__)
         self.parent.assertEqual(result.start_logits.shape			,    (self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape			,    (self.batch_size, self.seq_length))




     def 		snake_case_		(					self):
         A__					=       self.prepare_config_and_inputs()
         (
             (
             A__
         )			,						(
             A__
         )			,						(
             A__
         )			,						(
             A__
         )			,						(
             A__
         )			,						(
             A__
         )			,						(
             A__
         )			,						
         )					=       config_and_inputs
         A__					=       {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict







@require_tf
class      _UpperCAmelCase      (							A__      ,   A__      ,   unittest.TestCase  ):
     UpperCamelCase__		     =					(
         (
             TFDebertaVaModel,
             TFDebertaVaForMaskedLM,
             TFDebertaVaForQuestionAnswering,
             TFDebertaVaForSequenceClassification,
             TFDebertaVaForTokenClassification,
         )
         if is_tf_available()
         else ()
     )
     UpperCamelCase__		     =					(
         {
             '''feature-extraction''': TFDebertaVaModel,
             '''fill-mask''': TFDebertaVaForMaskedLM,
             '''question-answering''': TFDebertaVaForQuestionAnswering,
             '''text-classification''': TFDebertaVaForSequenceClassification,
             '''token-classification''': TFDebertaVaForTokenClassification,
             '''zero-shot''': TFDebertaVaForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )

     UpperCamelCase__		     =					False
     UpperCamelCase__		     =					False




     def 		snake_case_		(					self):
         A__					=       TFDebertaVaModelTester(self)
         A__					=       ConfigTester(self			,    config_class=a__			,    hidden_size=3_7)




     def 		snake_case_		(					self):
         self.config_tester.run_common_tests()




     def 		snake_case_		(					self):
         A__					=       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*a__)




     def 		snake_case_		(					self):
         A__					=       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*a__)




     def 		snake_case_		(					self):
         A__					=       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*a__)




     def 		snake_case_		(					self):
         A__					=       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*a__)




     def 		snake_case_		(					self):
         A__					=       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*a__)




     @slow
     def 		snake_case_		(					self):
         A__					=       TFDebertaVaModel.from_pretrained('''kamalkraj/deberta-v2-xlarge''')
         self.assertIsNotNone(a__)







@require_tf
class      _UpperCAmelCase      (							unittest.TestCase  ):




     @unittest.skip(reason='''Model not available yet''')
     def 		snake_case_		(					self):
         pass




     @slow
     def 		snake_case_		(					self):
         A__					=       TFDebertaVaModel.from_pretrained('''kamalkraj/deberta-v2-xlarge''')
         A__					=       tf.constant([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]])
         A__					=       tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
         A__					=       model(a__			,    attention_mask=a__)[0]

         A__					=       tf.constant(
             [[[0.2_3_5_6, 0.1_9_4_8, 0.0_3_6_9], [-0.1_0_6_3, 0.3_5_8_6, -0.5_1_5_2], [-0.6_3_9_9, -0.0_2_5_9, -0.2_5_2_5]]])
         tf.debugging.assert_near(output[:, 1:4, 1:4]			,    a__			,    atol=1e-4)
