





"""simple docstring"""





from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFDebertaVaForMaskedLM,
      TFDebertaVaForQuestionAnswering,
      TFDebertaVaForSequenceClassification,
      TFDebertaVaForTokenClassification,
      TFDebertaVaModel,
  )






class    _lowerCAmelCase  :
       """simple docstring"""




       def __init__(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase=1_3       ,					__UpperCAmelCase=7       ,					__UpperCAmelCase=True       ,					__UpperCAmelCase=True       ,					__UpperCAmelCase=True       ,					__UpperCAmelCase=True       ,					__UpperCAmelCase=9_9       ,					__UpperCAmelCase=3_2       ,					__UpperCAmelCase=2       ,					__UpperCAmelCase=4       ,					__UpperCAmelCase=3_7       ,					__UpperCAmelCase="gelu"       ,					__UpperCAmelCase=0.1       ,					__UpperCAmelCase=0.1       ,					__UpperCAmelCase=5_1_2       ,					__UpperCAmelCase=1_6       ,					__UpperCAmelCase=2       ,					__UpperCAmelCase=0.02       ,					__UpperCAmelCase=False       ,					__UpperCAmelCase=True       ,					__UpperCAmelCase="None"       ,					__UpperCAmelCase=3       ,					__UpperCAmelCase=4       ,					__UpperCAmelCase=None       ,					):



         '''simple docstring'''

         lowerCAmelCase__     :Tuple											=				parent
         lowerCAmelCase__     :int											=				batch_size
         lowerCAmelCase__     :Tuple											=				seq_length
         lowerCAmelCase__     :Optional[Any]											=				is_training
         lowerCAmelCase__     :int											=				use_input_mask
         lowerCAmelCase__     :Dict											=				use_token_type_ids
         lowerCAmelCase__     :List[str]											=				use_labels
         lowerCAmelCase__     :Optional[Any]											=				vocab_size
         lowerCAmelCase__     :Dict											=				hidden_size
         lowerCAmelCase__     :Optional[int]											=				num_hidden_layers
         lowerCAmelCase__     :List[Any]											=				num_attention_heads
         lowerCAmelCase__     :str											=				intermediate_size
         lowerCAmelCase__     :Optional[int]											=				hidden_act
         lowerCAmelCase__     :Tuple											=				hidden_dropout_prob
         lowerCAmelCase__     :int											=				attention_probs_dropout_prob
         lowerCAmelCase__     :int											=				max_position_embeddings
         lowerCAmelCase__     :Optional[int]											=				type_vocab_size
         lowerCAmelCase__     :Any											=				type_sequence_label_size
         lowerCAmelCase__     :Tuple											=				initializer_range
         lowerCAmelCase__     :Any											=				num_labels
         lowerCAmelCase__     :Tuple											=				num_choices
         lowerCAmelCase__     :int											=				relative_attention
         lowerCAmelCase__     :Tuple											=				position_biased_input
         lowerCAmelCase__     :Union[str, Any]											=				pos_att_type
         lowerCAmelCase__     :Tuple											=				scope






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :List[str]											=				ids_tensor([self.batch_size, self.seq_length]       ,					self.vocab_size )

         lowerCAmelCase__     :str											=				None
         if self.use_input_mask:
           lowerCAmelCase__     :Union[str, Any]											=				random_attention_mask([self.batch_size, self.seq_length] )

         lowerCAmelCase__     :int											=				None
         if self.use_token_type_ids:
           lowerCAmelCase__     :Optional[Any]											=				ids_tensor([self.batch_size, self.seq_length]       ,					self.type_vocab_size )

         lowerCAmelCase__     :int											=				None
         lowerCAmelCase__     :List[Any]											=				None
         lowerCAmelCase__     :Dict											=				None
         if self.use_labels:
           lowerCAmelCase__     :List[str]											=				ids_tensor([self.batch_size]       ,					self.type_sequence_label_size )
           lowerCAmelCase__     :Union[str, Any]											=				ids_tensor([self.batch_size, self.seq_length]       ,					self.num_labels )

         lowerCAmelCase__     :Dict											=				DebertaVaConfig(
             vocab_size=self.vocab_size       ,					hidden_size=self.hidden_size       ,					num_hidden_layers=self.num_hidden_layers       ,					num_attention_heads=self.num_attention_heads       ,					intermediate_size=self.intermediate_size       ,					hidden_act=self.hidden_act       ,					hidden_dropout_prob=self.hidden_dropout_prob       ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,					max_position_embeddings=self.max_position_embeddings       ,					type_vocab_size=self.type_vocab_size       ,					relative_attention=self.relative_attention       ,					position_biased_input=self.position_biased_input       ,					initializer_range=self.initializer_range       ,					return_dict=__UpperCAmelCase       ,					)

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[int]											=				TFDebertaVaModel(config=__UpperCAmelCase )
         lowerCAmelCase__     :List[Any]											=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

         lowerCAmelCase__     :Union[str, Any]											=				[input_ids, input_mask]
         lowerCAmelCase__     :Union[str, Any]											=				model(__UpperCAmelCase )

         lowerCAmelCase__     :Optional[Any]											=				model(__UpperCAmelCase )

         self.parent.assertEqual(result.last_hidden_state.shape       ,					(self.batch_size, self.seq_length, self.hidden_size) )






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[Any]											=				TFDebertaVaForMaskedLM(config=__UpperCAmelCase )
         lowerCAmelCase__     :Optional[int]											=				{
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }
         lowerCAmelCase__     :List[Any]											=				model(__UpperCAmelCase )
         self.parent.assertEqual(result.logits.shape       ,					(self.batch_size, self.seq_length, self.vocab_size) )






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :Dict											=				self.num_labels
         lowerCAmelCase__     :Dict											=				TFDebertaVaForSequenceClassification(config=__UpperCAmelCase )
         lowerCAmelCase__     :Optional[int]											=				{
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }

         lowerCAmelCase__     :Union[str, Any]											=				model(__UpperCAmelCase )
         self.parent.assertEqual(result.logits.shape       ,					(self.batch_size, self.num_labels) )






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :int											=				self.num_labels
         lowerCAmelCase__     :List[Any]											=				TFDebertaVaForTokenClassification(config=__UpperCAmelCase )
         lowerCAmelCase__     :Union[str, Any]											=				{
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }
         lowerCAmelCase__     :List[str]											=				model(__UpperCAmelCase )
         self.parent.assertEqual(result.logits.shape       ,					(self.batch_size, self.seq_length, self.num_labels) )






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase       ,					__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :List[Any]											=				TFDebertaVaForQuestionAnswering(config=__UpperCAmelCase )
         lowerCAmelCase__     :Optional[Any]											=				{
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }

         lowerCAmelCase__     :Optional[int]											=				model(__UpperCAmelCase )
         self.parent.assertEqual(result.start_logits.shape       ,					(self.batch_size, self.seq_length) )
         self.parent.assertEqual(result.end_logits.shape       ,					(self.batch_size, self.seq_length) )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Union[str, Any]											=				self.prepare_config_and_inputs()
         (
             (
             lowerCAmelCase__
         )				,							(
             lowerCAmelCase__
         )				,							(
             lowerCAmelCase__
         )				,							(
             lowerCAmelCase__
         )				,							(
             lowerCAmelCase__
         )				,							(
             lowerCAmelCase__
         )				,							(
             lowerCAmelCase__
         )				,							
         )     :Tuple											=				config_and_inputs
         lowerCAmelCase__     :Tuple											=				{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
         return config, inputs_dict






@require_tf
class    _lowerCAmelCase  (	a    ,  a    ,  unittest.TestCase			):
       """simple docstring"""




       __magic_name__	:List[str]						    =   (
           (
               TFDebertaVaModel,
               TFDebertaVaForMaskedLM,
               TFDebertaVaForQuestionAnswering,
               TFDebertaVaForSequenceClassification,
               TFDebertaVaForTokenClassification,
           )
           if is_tf_available()
           else ()
       )
       __magic_name__	:List[Any]						    =   (
           {
               """feature-extraction""": TFDebertaVaModel,
               """fill-mask""": TFDebertaVaForMaskedLM,
               """question-answering""": TFDebertaVaForQuestionAnswering,
               """text-classification""": TFDebertaVaForSequenceClassification,
               """token-classification""": TFDebertaVaForTokenClassification,
               """zero-shot""": TFDebertaVaForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )

       __magic_name__	:List[str]						    =   False
       __magic_name__	:Optional[Any]						    =   False






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :List[str]											=				TFDebertaVaModelTester(self )
         lowerCAmelCase__     :str											=				ConfigTester(self       ,					config_class=__UpperCAmelCase       ,					hidden_size=3_7 )






       def 	snake_case		(       self ):



         '''simple docstring'''

         self.config_tester.run_common_tests()






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :int											=				self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :List[str]											=				self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[Any]											=				self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Tuple											=				self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Tuple											=				self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase )






       @slow
       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[int]											=				TFDebertaVaModel.from_pretrained('kamalkraj/deberta-v2-xlarge' )
         self.assertIsNotNone(__UpperCAmelCase )






@require_tf
class    _lowerCAmelCase  (	unittest.TestCase			):
       """simple docstring"""




       @unittest.skip(reason='Model not available yet' )
       def 	snake_case		(       self ):



         '''simple docstring'''

         pass






       @slow
       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :int											=				TFDebertaVaModel.from_pretrained('kamalkraj/deberta-v2-xlarge' )
         lowerCAmelCase__     :Optional[int]											=				tf.constant([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]] )
         lowerCAmelCase__     :Optional[Any]											=				tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
         lowerCAmelCase__     :Union[str, Any]											=				model(__UpperCAmelCase       ,					attention_mask=__UpperCAmelCase )[0]

         lowerCAmelCase__     :Optional[Any]											=				tf.constant(
             [[[0.23_56, 0.19_48, 0.03_69], [-0.10_63, 0.35_86, -0.51_52], [-0.63_99, -0.02_59, -0.25_25]]] )
         tf.debugging.assert_near(output[:, 1:4, 1:4]       ,					__UpperCAmelCase       ,					atol=1E-4 )




import os

from bleurt import score  # From: git+https://github.com/google-research/bleurt.git

import datasets


SCREAMING_SNAKE_CASE__     					=			datasets.logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__     					=			"\\n@inproceedings{bleurt,\n  title={BLEURT: Learning Robust Metrics for Text Generation},\n  author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh},\n  booktitle={ACL},\n  year={2020},\n  url={https://arxiv.org/abs/2004.04696}\n}\n"

SCREAMING_SNAKE_CASE__     					=			"\\nBLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018)\nand then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune\nit for your specific application (the latter is expected to perform better).\n\nSee the project's README at https://github.com/google-research/bleurt#readme for more information.\n"

SCREAMING_SNAKE_CASE__     					=			"\nBLEURT score.\n\nArgs:\n    `predictions` (list of str): prediction/candidate sentences\n    `references` (list of str): reference sentences\n    `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None.\n\nReturns:\n    'scores': List of scores.\nExamples:\n\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> bleurt = datasets.load_metric(\"bleurt\")\n    >>> results = bleurt.compute(predictions=predictions, references=references)\n    >>> print([round(v, 2) for v in results[\"scores\"]])\n    [1.03, 1.04]\n"

SCREAMING_SNAKE_CASE__     					=			{
    "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip",
    "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip",
    "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip",
    "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip",
    "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip",
    "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip",
    "BLEURT-20-D3": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip",
    "BLEURT-20-D6": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip",
    "BLEURT-20-D12": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip",
    "BLEURT-20": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip",
}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,		_KWARGS_DESCRIPTION							)
class 						_UpperCAmelCase      (     datasets.Metric							):






		def _snake_case		(						self				:     List[str]):
				return datasets.MetricInfo(
				    description=_DESCRIPTION   ,  citation=_CITATION   ,  homepage="https://github.com/google-research/bleurt"   ,  inputs_description=_KWARGS_DESCRIPTION   ,  features=datasets.Features(
				        {
				            "predictions": datasets.Value("string"   ,  id="sequence"),
				            "references": datasets.Value("string"   ,  id="sequence"),
				        })   ,  codebase_urls=["https://github.com/google-research/bleurt"]   ,  reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"]   ,  )






		def _snake_case		(						self				:     Any   ,  UpperCAmelCase				:     Any):
				# check that config name specifies a valid BLEURT model
				if self.config_name == "default":
						logger.warning(
						    "Using default BLEURT-Base checkpoint for sequence maximum length 128. "
						    "You can use a bigger model for better results with e.g.: datasets.load_metric('bleurt', 'bleurt-large-512').")
						SCREAMING_SNAKE_CASE_				:str				     =						"bleurt-base-128"

				if self.config_name.lower() in CHECKPOINT_URLS:
						SCREAMING_SNAKE_CASE_				:Dict				     =						self.config_name.lower()

				elif self.config_name.upper() in CHECKPOINT_URLS:
						SCREAMING_SNAKE_CASE_				:Dict				     =						self.config_name.upper()

				else:
						raise KeyError(
						    F"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}")

				# download the model checkpoint specified by self.config_name and set up the scorer
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name])
				SCREAMING_SNAKE_CASE_				:Dict				     =						score.BleurtScorer(os.path.join(UpperCAmelCase   ,  UpperCAmelCase))







		def _snake_case		(						self				:     Union[str, Any]   ,  UpperCAmelCase				:     List[str]   ,  UpperCAmelCase				:     Union[str, Any]):
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						self.scorer.score(references=UpperCAmelCase   ,  candidates=UpperCAmelCase)
				return {"scores": scores}






