


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_pegasus import PegasusTokenizer
else:
			_snake_case     :		Optional[Any]						=				None


_snake_case     :		int						=				logging.get_logger(__name__)


_snake_case     :		Union[str, Any]						=				"▁"

_snake_case     :		Union[str, Any]						=				{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

_snake_case     :		Dict						=				{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"},
    "tokenizer_file": {
        "google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"
    },
}

_snake_case     :		Tuple						=				{
    "google/pegasus-xsum": 512,
}

class 				a		(_UpperCAmelCase      ):




							"""simple docstring"""


							__UpperCAmelCase    :							List[Any]       =  VOCAB_FILES_NAMES
							__UpperCAmelCase    :							Optional[int]       =  PRETRAINED_VOCAB_FILES_MAP
							__UpperCAmelCase    :							Tuple       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__UpperCAmelCase    :							Optional[int]       =  PegasusTokenizer
							__UpperCAmelCase    :							str       =  ["input_ids", "attention_mask"]

							def __init__(							self       :  Optional[int]    ,	lowerCamelCase       :  Optional[Any]=None    ,	lowerCamelCase       :  List[str]=None    ,	lowerCamelCase       :  Optional[Any]="<pad>"    ,	lowerCamelCase       :  Dict="</s>"    ,	lowerCamelCase       :  Optional[int]="<unk>"    ,	lowerCamelCase       :  Dict="<mask_2>"    ,	lowerCamelCase       :  Tuple="<mask_1>"    ,	lowerCamelCase       :  List[Any]=None    ,	lowerCamelCase       :  Union[str, Any]=103    ,	**lowerCamelCase       :  Dict    ,	)	->    List[str]:
														__snake_case		:			Any							 =   offset

														if additional_special_tokens is not None:
																					if not isinstance(A_    ,	A_				):
																												raise TypeError(
																												    F'additional_special_tokens should be of type {type(A_				)}, but is'
																												    F' {type(A_				)}'				)

																					__snake_case		:			Union[str, Any]							 =   (
																					    ([mask_token_sent] + additional_special_tokens)
																					    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
																					    else additional_special_tokens
																					)
																					# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
																					additional_special_tokens_extended += [
																					    F'<unk_{i}>' for i in range(len(A_				)    ,	self.offset - 1				)
																					]

																					if len(set(A_				)				) != len(A_				):
																												raise ValueError(
																												    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
																												    F' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'				)
																					__snake_case		:			Union[str, Any]							 =   additional_special_tokens_extended
														else:
																					__snake_case		:			Any							 =   [mask_token_sent] if mask_token_sent is not None else []
																					additional_special_tokens += [F'<unk_{i}>' for i in range(2    ,	self.offset				)]

														super().__init__(
														    A_    ,	tokenizer_file=A_    ,	pad_token=A_    ,	eos_token=A_    ,	unk_token=A_    ,	mask_token=A_    ,	mask_token_sent=A_    ,	offset=A_    ,	additional_special_tokens=A_    ,	**A_    ,	)
														__snake_case		:			List[str]							 =   vocab_file
														__snake_case		:			List[Any]							 =   False if not self.vocab_file else True

							def     __snake_case					(							self       :  Union[str, Any]    ,	lowerCamelCase       :  List[str]				)	->    str:
														__snake_case		:			int							 =   set(self.all_special_ids				)  # call it once instead of inside list comp
														all_special_ids.remove(self.unk_token_id				)  # <unk> is only sometimes special

														if all_special_ids != set(range(len(self.additional_special_tokens				) + 3				)				):
																					raise ValueError(
																					    "There should be 3 special tokens: mask_token, pad_token, and eos_token +"
																					    F' {len(self.additional_special_tokens				)} additional_special_tokens, but got {all_special_ids}'				)

														return [1 if x in all_special_ids else 0 for x in seq]

							def     __snake_case					(							self       :  Union[str, Any]    ,	lowerCamelCase       :  int    ,	lowerCamelCase       :  str = None    ,	lowerCamelCase       :  Optional[int] = False				)	->    List[int]:





														if already_has_special_tokens:
																					return self._special_token_mask(A_				)
														elif token_ids_a is None:
																					return self._special_token_mask(A_				) + [1]
														else:
																					return self._special_token_mask(token_ids_a + token_ids_a				) + [1]

							def     __snake_case					(							self       :  str    ,	lowerCamelCase       :  Optional[Any]    ,	lowerCamelCase       :  Tuple=None				)	->    List[int]:





														if token_ids_a is None:
																					return token_ids_a + [self.eos_token_id]
														# We don't expect to process pairs, but leave the pair logic for API consistency
														return token_ids_a + token_ids_a + [self.eos_token_id]






							def     __snake_case					(							self       :  Union[str, Any]    ,	lowerCamelCase       :  Dict    ,	lowerCamelCase       :  int = None				)	->    Tuple[str]:





														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																					    "tokenizer."				)

														if not os.path.isdir(A_				):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory'				)
																					return
														__snake_case		:			Dict							 =   os.path.join(
														    A_    ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

														if os.path.abspath(self.vocab_file				) != os.path.abspath(A_				):
																					copyfile(self.vocab_file    ,	A_				)

														return (out_vocab_file,)




import unittest

from transformers import DebertaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     DebertaForMaskedLM,
     DebertaForQuestionAnswering,
     DebertaForSequenceClassification,
     DebertaForTokenClassification,
     DebertaModel,
 )
 from transformers.models.deberta.modeling_deberta import DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST







class   __lowercase      (_UpperCAmelCase ):
 def __init__(							self      ,							A_      ,							A_=13      ,							A_=7      ,							A_=True      ,							A_=True      ,							A_=True      ,							A_=True      ,							A_=99      ,							A_=32      ,							A_=5      ,							A_=4      ,							A_=37      ,							A_="gelu"      ,							A_=0.1      ,							A_=0.1      ,							A_=512      ,							A_=16      ,							A_=2      ,							A_=0.02      ,							A_=False      ,							A_=True      ,							A_="None"      ,							A_=3      ,							A_=4      ,							A_=None      ,							)					->List[Any]:



       '''simple docstring'''





       __lowerCAmelCase					:						Union[str, Any]					=					parent
       __lowerCAmelCase					:						List[str]					=					batch_size
       __lowerCAmelCase					:						Dict					=					seq_length
       __lowerCAmelCase					:						List[Any]					=					is_training
       __lowerCAmelCase					:						List[Any]					=					use_input_mask
       __lowerCAmelCase					:						Optional[int]					=					use_token_type_ids
       __lowerCAmelCase					:						Tuple					=					use_labels
       __lowerCAmelCase					:						str					=					vocab_size
       __lowerCAmelCase					:						int					=					hidden_size
       __lowerCAmelCase					:						Any					=					num_hidden_layers
       __lowerCAmelCase					:						Any					=					num_attention_heads
       __lowerCAmelCase					:						Dict					=					intermediate_size
       __lowerCAmelCase					:						int					=					hidden_act
       __lowerCAmelCase					:						int					=					hidden_dropout_prob
       __lowerCAmelCase					:						Any					=					attention_probs_dropout_prob
       __lowerCAmelCase					:						List[str]					=					max_position_embeddings
       __lowerCAmelCase					:						Union[str, Any]					=					type_vocab_size
       __lowerCAmelCase					:						Union[str, Any]					=					type_sequence_label_size
       __lowerCAmelCase					:						Optional[int]					=					initializer_range
       __lowerCAmelCase					:						int					=					num_labels
       __lowerCAmelCase					:						int					=					num_choices
       __lowerCAmelCase					:						List[str]					=					relative_attention
       __lowerCAmelCase					:						Union[str, Any]					=					position_biased_input
       __lowerCAmelCase					:						int					=					pos_att_type
       __lowerCAmelCase					:						List[Any]					=					scope
 def 	UpperCamelCase__  (							self					)					->Dict:



       '''simple docstring'''





       __lowerCAmelCase					:						Dict					=					ids_tensor([self.batch_size, self.seq_length]      ,							self.vocab_size					)

       __lowerCAmelCase					:						int					=					None
       if self.use_input_mask:
             __lowerCAmelCase					:						List[Any]					=					ids_tensor([self.batch_size, self.seq_length]      ,							vocab_size=2					)

       __lowerCAmelCase					:						List[str]					=					None
       if self.use_token_type_ids:
             __lowerCAmelCase					:						Tuple					=					ids_tensor([self.batch_size, self.seq_length]      ,							self.type_vocab_size					)

       __lowerCAmelCase					:						Union[str, Any]					=					None
       __lowerCAmelCase					:						int					=					None
       __lowerCAmelCase					:						List[str]					=					None
       if self.use_labels:
             __lowerCAmelCase					:						str					=					ids_tensor([self.batch_size]      ,							self.type_sequence_label_size					)
             __lowerCAmelCase					:						Dict					=					ids_tensor([self.batch_size, self.seq_length]      ,							self.num_labels					)
             __lowerCAmelCase					:						Union[str, Any]					=					ids_tensor([self.batch_size]      ,							self.num_choices					)

       __lowerCAmelCase					:						Tuple					=					self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 def 	UpperCamelCase__  (							self					)					->List[Any]:



       '''simple docstring'''





       return DebertaConfig(
           vocab_size=self.vocab_size      ,							hidden_size=self.hidden_size      ,							num_hidden_layers=self.num_hidden_layers      ,							num_attention_heads=self.num_attention_heads      ,							intermediate_size=self.intermediate_size      ,							hidden_act=self.hidden_act      ,							hidden_dropout_prob=self.hidden_dropout_prob      ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,							max_position_embeddings=self.max_position_embeddings      ,							type_vocab_size=self.type_vocab_size      ,							initializer_range=self.initializer_range      ,							relative_attention=self.relative_attention      ,							position_biased_input=self.position_biased_input      ,							pos_att_type=self.pos_att_type      ,							)
 def 	UpperCamelCase__  (							self					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					self.get_config()
       __lowerCAmelCase					:						Dict					=					300
       return config
 def 	UpperCamelCase__  (							self      ,							A_					)					->Union[str, Any]:



       '''simple docstring'''





       self.parent.assertListEqual(list(result.loss.size()					)      ,							[]					)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_					)					->Any:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[Any]					=					DebertaModel(config=A_					)
       model.to(A_					)
       model.eval()
       __lowerCAmelCase					:						str					=					model(A_      ,							attention_mask=A_      ,							token_type_ids=A_					)[0]
       __lowerCAmelCase					:						Any					=					model(A_      ,							token_type_ids=A_					)[0]
       __lowerCAmelCase					:						List[str]					=					model(A_					)[0]

       self.parent.assertListEqual(list(sequence_output.size()					)      ,							[self.batch_size, self.seq_length, self.hidden_size]					)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_					)					->int:



       '''simple docstring'''





       __lowerCAmelCase					:						Tuple					=					DebertaForMaskedLM(config=A_					)
       model.to(A_					)
       model.eval()
       __lowerCAmelCase					:						Union[str, Any]					=					model(A_      ,							attention_mask=A_      ,							token_type_ids=A_      ,							labels=A_					)

       self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.seq_length, self.vocab_size)					)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_					)					->Any:



       '''simple docstring'''





       __lowerCAmelCase					:						Any					=					self.num_labels
       __lowerCAmelCase					:						Tuple					=					DebertaForSequenceClassification(A_					)
       model.to(A_					)
       model.eval()
       __lowerCAmelCase					:						Union[str, Any]					=					model(A_      ,							attention_mask=A_      ,							token_type_ids=A_      ,							labels=A_					)
       self.parent.assertListEqual(list(result.logits.size()					)      ,							[self.batch_size, self.num_labels]					)
       self.check_loss_output(A_					)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_					)					->Dict:



       '''simple docstring'''





       __lowerCAmelCase					:						List[Any]					=					self.num_labels
       __lowerCAmelCase					:						Optional[int]					=					DebertaForTokenClassification(config=A_					)
       model.to(A_					)
       model.eval()
       __lowerCAmelCase					:						Tuple					=					model(A_      ,							attention_mask=A_      ,							token_type_ids=A_      ,							labels=A_					)
       self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.seq_length, self.num_labels)					)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						List[str]					=					DebertaForQuestionAnswering(config=A_					)
       model.to(A_					)
       model.eval()
       __lowerCAmelCase					:						int					=					model(
           A_      ,							attention_mask=A_      ,							token_type_ids=A_      ,							start_positions=A_      ,							end_positions=A_      ,							)
       self.parent.assertEqual(result.start_logits.shape      ,							(self.batch_size, self.seq_length)					)
       self.parent.assertEqual(result.end_logits.shape      ,							(self.batch_size, self.seq_length)					)





 def 	UpperCamelCase__  (							self					)					->List[str]:



       '''simple docstring'''





       __lowerCAmelCase					:						Any					=					self.prepare_config_and_inputs()
       (
           (
           __lowerCAmelCase
       ),							(
           __lowerCAmelCase
       ),							(
           __lowerCAmelCase
       ),							(
           __lowerCAmelCase
       ),							(
           __lowerCAmelCase
       ),							(
           __lowerCAmelCase
       ),							(
           __lowerCAmelCase
       ),							
       )					:						Tuple					=					config_and_inputs
       __lowerCAmelCase					:						Tuple					=					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict







@require_torch
class   __lowercase      (_UpperCAmelCase		,   _UpperCAmelCase		,   unittest.TestCase ):
 _UpperCamelCase			  =  (
     (
         DebertaModel,
         DebertaForMaskedLM,
         DebertaForSequenceClassification,
         DebertaForTokenClassification,
         DebertaForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 _UpperCamelCase			  =  (
     {
         """feature-extraction""": DebertaModel,
         """fill-mask""": DebertaForMaskedLM,
         """question-answering""": DebertaForQuestionAnswering,
         """text-classification""": DebertaForSequenceClassification,
         """token-classification""": DebertaForTokenClassification,
         """zero-shot""": DebertaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )

 _UpperCamelCase			  =  True
 _UpperCamelCase			  =  False
 _UpperCamelCase			  =  False
 _UpperCamelCase			  =  False
 _UpperCamelCase			  =  False
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					DebertaModelTester(self					)
       __lowerCAmelCase					:						List[Any]					=					ConfigTester(self      ,							config_class=A_      ,							hidden_size=37					)
 def 	UpperCamelCase__  (							self					)					->Optional[Any]:



       '''simple docstring'''





       self.config_tester.run_common_tests()
 def 	UpperCamelCase__  (							self					)					->List[Any]:



       '''simple docstring'''





       __lowerCAmelCase					:						Dict					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_model(*A_					)
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       __lowerCAmelCase					:						Any					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_sequence_classification(*A_					)
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       __lowerCAmelCase					:						Any					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_masked_lm(*A_					)
 def 	UpperCamelCase__  (							self					)					->Optional[int]:



       '''simple docstring'''





       __lowerCAmelCase					:						List[Any]					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_question_answering(*A_					)
 def 	UpperCamelCase__  (							self					)					->List[Any]:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_token_classification(*A_					)





 @slow
 def 	UpperCamelCase__  (							self					)					->Tuple:



       '''simple docstring'''





       for model_name in DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __lowerCAmelCase					:						Optional[int]					=					DebertaModel.from_pretrained(A_					)
             self.assertIsNotNone(A_					)







@require_torch
@require_sentencepiece
@require_tokenizers
class   __lowercase      (unittest.TestCase ):
 @unittest.skip(reason='''Model not available yet'''					)
 def 	UpperCamelCase__  (							self					)					->Dict:



       '''simple docstring'''





       pass





 @slow
 def 	UpperCamelCase__  (							self					)					->Tuple:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					DebertaModel.from_pretrained('''microsoft/deberta-base'''					)

       __lowerCAmelCase					:						Tuple					=					torch.tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]					)
       __lowerCAmelCase					:						Union[str, Any]					=					torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]					)
       with torch.no_grad():
             __lowerCAmelCase					:						Optional[int]					=					model(A_      ,							attention_mask=A_					)[0]
       # compare the actual values for a slice.
       __lowerCAmelCase					:						Optional[Any]					=					torch.tensor(
           [[[-0.5_986, -0.8_055, -0.8_462], [1.4_484, -0.9_348, -0.8_059], [0.3_123, 0.0_032, -1.4_131]]]					)
       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,							A_      ,							atol=1e-4					)      ,							f"""{output[:, 1:4, 1:4]}"""					)


