


'''simple docstring'''







from __future__ import annotations

import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import is_tf_available, is_vision_available

from ...test_modeling_tf_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_tf_bert import TFBertModelTester
from ..clip.test_modeling_tf_clip import TFCLIPVisionModelTester
from ..deit.test_modeling_tf_deit import TFDeiTModelTester
from ..roberta.test_modeling_tf_roberta import TFRobertaModelTester
from ..vit.test_modeling_tf_vit import TFViTModelTester


if is_tf_available():
   from transformers import (
       TFBertModel,
       TFCLIPVisionModel,
       TFDeiTModel,
       TFRobertaModel,
       TFVisionTextDualEncoderModel,
       TFViTModel,
       VisionTextDualEncoderConfig,
   )

if is_vision_available():
   from PIL import Image

   from transformers import VisionTextDualEncoderProcessor


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Union[str, Any]  )  ->			Union[str, Any]:
       if isinstance(lowercase__    ,collections.abc.Iterable  ):
              return x
       return (x, x)







@require_tf
class A__       :






      def __UpperCAmelCase       ( self			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Any     ,						SCREAMING_SNAKE_CASE			:Any	)       ->					Any:




             '''simple docstring'''
             pass






      def __UpperCAmelCase       ( self			:Dict	)       ->					int:




             '''simple docstring'''
             pass






      def __UpperCAmelCase       ( self			:str	)       ->					Union[str, Any]:




             '''simple docstring'''
             pass






      def __UpperCAmelCase       ( self			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:int     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Any=None     ,						**SCREAMING_SNAKE_CASE			:List[str]	)       ->					Any:




             '''simple docstring'''
             _a    : Tuple     					=VisionTextDualEncoderConfig.from_vision_text_configs(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             _a    : int     					=TFVisionTextDualEncoderModel(SCREAMING_SNAKE_CASE	)

             _a    : Optional[Any]     					=model(input_ids=SCREAMING_SNAKE_CASE     ,						pixel_values=SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE	)

             self.assertEqual(output["""text_embeds"""].shape     ,						(input_ids.shape[0], config.projection_dim)	)
             self.assertEqual(output["""image_embeds"""].shape     ,						(pixel_values.shape[0], config.projection_dim)	)






      def __UpperCAmelCase       ( self			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:Any     ,						SCREAMING_SNAKE_CASE			:int     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:List[Any]=None     ,						**SCREAMING_SNAKE_CASE			:List[Any]	)       ->					List[Any]:




             '''simple docstring'''
             _a    : Dict     					=self.get_vision_text_model(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             _a    : Optional[Any]     					=TFVisionTextDualEncoderModel(vision_model=SCREAMING_SNAKE_CASE     ,						text_model=SCREAMING_SNAKE_CASE	)

             _a    : List[str]     					=model(input_ids=SCREAMING_SNAKE_CASE     ,						pixel_values=SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE	)

             self.assertEqual(output["""text_embeds"""].shape     ,						(input_ids.shape[0], model.config.projection_dim)	)
             self.assertEqual(output["""image_embeds"""].shape     ,						(pixel_values.shape[0], model.config.projection_dim)	)






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:int     ,						SCREAMING_SNAKE_CASE			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=None     ,						**SCREAMING_SNAKE_CASE			:Union[str, Any]	)       ->					Any:




             '''simple docstring'''
             _a    : List[Any]     					=self.get_vision_text_model(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             _a    : Tuple     					={'vision_model': vision_model, 'text_model': text_model}
             _a    : Dict     					=TFVisionTextDualEncoderModel.from_vision_text_pretrained(**SCREAMING_SNAKE_CASE	)

             _a    : Dict     					=model(input_ids=SCREAMING_SNAKE_CASE     ,						pixel_values=SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE	)

             self.assertEqual(output["""text_embeds"""].shape     ,						(input_ids.shape[0], model.config.projection_dim)	)
             self.assertEqual(output["""image_embeds"""].shape     ,						(pixel_values.shape[0], model.config.projection_dim)	)






      def __UpperCAmelCase       ( self			:Dict     ,						SCREAMING_SNAKE_CASE			:Any     ,						SCREAMING_SNAKE_CASE			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=None     ,						**SCREAMING_SNAKE_CASE			:str	)       ->					Union[str, Any]:




             '''simple docstring'''
             _a    : int     					=self.get_vision_text_model(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             _a    : List[Any]     					=TFVisionTextDualEncoderModel(vision_model=SCREAMING_SNAKE_CASE     ,						text_model=SCREAMING_SNAKE_CASE	)

             _a    : Union[str, Any]     					=model(input_ids=SCREAMING_SNAKE_CASE     ,						pixel_values=SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE	)
             _a    : Union[str, Any]     					=output[0].numpy()

             with tempfile.TemporaryDirectory() as tmpdirname:
                    model.save_pretrained(SCREAMING_SNAKE_CASE	)
                    _a    : List[Any]     					=TFVisionTextDualEncoderModel.from_pretrained(SCREAMING_SNAKE_CASE	)

                    _a    : Tuple     					=model(input_ids=SCREAMING_SNAKE_CASE     ,						pixel_values=SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE	)
                    _a    : List[str]     					=after_output[0].numpy()
                    _a    : Optional[int]     					=np.amax(np.abs(out_a - out_a	)	)
                    self.assertLessEqual(SCREAMING_SNAKE_CASE     ,						1e-5	)






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Dict     ,						SCREAMING_SNAKE_CASE			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=None     ,						**SCREAMING_SNAKE_CASE			:List[Any]	)       ->					List[Any]:




             '''simple docstring'''
             _a    : List[Any]     					=self.get_vision_text_model(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             _a    : List[str]     					=TFVisionTextDualEncoderModel(vision_model=SCREAMING_SNAKE_CASE     ,						text_model=SCREAMING_SNAKE_CASE	)

             _a    : Tuple     					=model(
                 input_ids=SCREAMING_SNAKE_CASE     ,						pixel_values=SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE     ,						output_attentions=SCREAMING_SNAKE_CASE	)

             _a    : List[Any]     					=output.vision_model_output.attentions
             self.assertEqual(len(SCREAMING_SNAKE_CASE	)     ,						vision_config.num_hidden_layers	)

             # in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
             _a    : Any     					=to_atuple(vision_model.config.image_size	)
             _a    : List[Any]     					=to_atuple(vision_model.config.patch_size	)
             _a    : Optional[int]     					=(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
             _a    : Tuple     					=num_patches + 1
             self.assertEqual(vision_attentions[0].shape[-3:]     ,						(vision_config.num_attention_heads, seq_len, seq_len)	)

             _a    : Union[str, Any]     					=output.text_model_output.attentions
             self.assertEqual(len(SCREAMING_SNAKE_CASE	)     ,						text_config.num_hidden_layers	)

             self.assertEqual(
                 text_attentions[0].shape[-3:]     ,						(text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])     ,						)






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]	)       ->					Dict:




             '''simple docstring'''
             _a    : Tuple     					=np.abs((a - b)	).max()
             self.assertLessEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE     ,						f"Difference between torch and flax is {diff} (>= {tol})."	)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Union[str, Any]:




             '''simple docstring'''
             _a    : Any     					=self.prepare_config_and_inputs()
             self.check_vision_text_dual_encoder_model(**SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					Tuple:




             '''simple docstring'''
             _a    : Optional[Any]     					=self.prepare_config_and_inputs()
             self.check_model_from_pretrained_configs(**SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					Any:




             '''simple docstring'''
             _a    : Optional[int]     					=self.prepare_config_and_inputs()
             self.check_vision_text_dual_encoder_from_pretrained(**SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:int	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : List[Any]     					=self.prepare_config_and_inputs()
             self.check_save_load(**SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:str	)       ->					Dict:




             '''simple docstring'''
             _a    : Any     					=self.prepare_config_and_inputs()
             self.check_vision_text_output_attention(**SCREAMING_SNAKE_CASE	)






      @slow
      def __UpperCAmelCase       ( self			:str	)       ->					List[Any]:




             '''simple docstring'''
             _a    : int     					=self.get_pretrained_model_and_inputs()

             _a    : str     					=model_a(**SCREAMING_SNAKE_CASE	)
             _a    : List[Any]     					=outputs[0].numpy()

             with tempfile.TemporaryDirectory() as tmp_dirname:
                    model_a.save_pretrained(SCREAMING_SNAKE_CASE	)
                    _a    : List[Any]     					=TFVisionTextDualEncoderModel.from_pretrained(SCREAMING_SNAKE_CASE	)

                    _a    : Union[str, Any]     					=model_a(**SCREAMING_SNAKE_CASE	)
                    _a    : Optional[Any]     					=after_outputs[0].numpy()
                    _a    : Tuple     					=np.amax(np.abs(out_a - out_a	)	)
                    self.assertLessEqual(SCREAMING_SNAKE_CASE     ,						1e-5	)







@require_tf
class A__       (						UpperCAmelCase__ ,     unittest.TestCase      ):






      def __UpperCAmelCase       ( self			:str	)       ->					List[Any]:




             '''simple docstring'''
             _a    : Optional[Any]     					=TFVisionTextDualEncoderModel.from_vision_text_pretrained(
                 """hf-internal-testing/tiny-random-vit"""     ,						"""hf-internal-testing/tiny-random-bert"""	)
             _a    : Tuple     					=1_3
             _a    : Tuple     					=floats_tensor(
                 [
                     batch_size,
                     model.vision_model.config.num_channels,
                     model.vision_model.config.image_size,
                     model.vision_model.config.image_size,
                 ]	)
             _a    : Dict     					=ids_tensor([batch_size, 4]     ,						model.text_model.config.vocab_size	)
             _a    : int     					=random_attention_mask([batch_size, 4]	)
             _a    : Any     					={'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

             return model, inputs






      def __UpperCAmelCase       ( self			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:int	)       ->					int:




             '''simple docstring'''
             _a    : int     					=TFViTModel(SCREAMING_SNAKE_CASE     ,						name="""vision_model"""	)
             _a    : Union[str, Any]     					=TFBertModel(SCREAMING_SNAKE_CASE     ,						name="""text_model"""	)
             return vision_model, text_model






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Dict:




             '''simple docstring'''
             _a    : Optional[int]     					=TFViTModelTester(self	)
             _a    : str     					=TFBertModelTester(self	)
             _a    : int     					=vit_model_tester.prepare_config_and_inputs()
             _a    : str     					=bert_model_tester.prepare_config_and_inputs()

             _a    : Dict     					=vision_config_and_inputs

             (
                 _a 
             )    : Union[str, Any]     					=text_config_and_inputs

             return {
                 "text_config": text_config,
                 "vision_config": vision_config,
                 "pixel_values": pixel_values,
                 "attention_mask": input_mask,
                 "input_ids": input_ids,
                 "text_token_type_ids": token_type_ids,
                 "text_sequence_labels": sequence_labels,
                 "text_token_labels": token_labels,
                 "text_choice_labels": choice_labels,
             }







@require_tf
class A__       (						UpperCAmelCase__ ,     unittest.TestCase      ):






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					int:




             '''simple docstring'''
             # DeiT repo doesn't have TF weights, but we don't actually use the weights at all so let's
             # just reinitialize it.
             _a    : int     					=TFVisionTextDualEncoderModel.from_vision_text_pretrained(
                 """Rocketknight1/tiny-random-deit-tf"""     ,						"""hf-internal-testing/tiny-random-roberta"""	)
             _a    : Tuple     					=1_3
             _a    : Tuple     					=floats_tensor(
                 [
                     batch_size,
                     model.vision_model.config.num_channels,
                     model.vision_model.config.image_size,
                     model.vision_model.config.image_size,
                 ]	)
             _a    : Tuple     					=ids_tensor([batch_size, 4]     ,						model.text_model.config.vocab_size	)
             _a    : str     					=random_attention_mask([batch_size, 4]	)
             _a    : Optional[Any]     					={'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

             return model, inputs






      def __UpperCAmelCase       ( self			:int     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Any     ,						SCREAMING_SNAKE_CASE			:str=None     ,						**SCREAMING_SNAKE_CASE			:Optional[int]	)       ->					List[str]:




             '''simple docstring'''
             _a    : Union[str, Any]     					=self.get_vision_text_model(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             _a    : Dict     					=TFVisionTextDualEncoderModel(vision_model=SCREAMING_SNAKE_CASE     ,						text_model=SCREAMING_SNAKE_CASE	)

             _a    : List[Any]     					=model(
                 input_ids=SCREAMING_SNAKE_CASE     ,						pixel_values=SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE     ,						output_attentions=SCREAMING_SNAKE_CASE	)

             _a    : Union[str, Any]     					=output.vision_model_output.attentions
             self.assertEqual(len(SCREAMING_SNAKE_CASE	)     ,						vision_config.num_hidden_layers	)

             # in DEiT, the seq_len equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
             _a    : Any     					=to_atuple(vision_model.config.image_size	)
             _a    : Union[str, Any]     					=to_atuple(vision_model.config.patch_size	)
             _a    : Union[str, Any]     					=(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
             _a    : Optional[int]     					=num_patches + 2
             self.assertEqual(vision_attentions[0].shape[-3:]     ,						(vision_config.num_attention_heads, seq_len, seq_len)	)

             _a    : List[Any]     					=output.text_model_output.attentions
             self.assertEqual(len(SCREAMING_SNAKE_CASE	)     ,						text_config.num_hidden_layers	)

             self.assertEqual(
                 text_attentions[0].shape[-3:]     ,						(text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])     ,						)






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:Dict     ,						SCREAMING_SNAKE_CASE			:Dict	)       ->					Any:




             '''simple docstring'''
             _a    : List[Any]     					=TFDeiTModel(SCREAMING_SNAKE_CASE     ,						name="""vision_model"""	)
             _a    : List[str]     					=TFRobertaModel(SCREAMING_SNAKE_CASE     ,						name="""text_model"""	)
             return vision_model, text_model






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : Any     					=TFDeiTModelTester(self	)
             _a    : Union[str, Any]     					=TFRobertaModelTester(self	)
             _a    : Optional[int]     					=vit_model_tester.prepare_config_and_inputs()
             _a    : int     					=bert_model_tester.prepare_config_and_inputs()

             _a    : Dict     					=vision_config_and_inputs

             (
                 _a 
             )    : Dict     					=text_config_and_inputs

             return {
                 "text_config": text_config,
                 "vision_config": vision_config,
                 "pixel_values": pixel_values,
                 "attention_mask": input_mask,
                 "input_ids": input_ids,
                 "text_token_type_ids": token_type_ids,
                 "text_sequence_labels": sequence_labels,
                 "text_token_labels": token_labels,
                 "text_choice_labels": choice_labels,
             }







@require_tf
class A__       (						UpperCAmelCase__ ,     unittest.TestCase      ):






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					Any:




             '''simple docstring'''
             _a    : Dict     					=TFVisionTextDualEncoderModel.from_vision_text_pretrained(
                 """Rocketknight1/tiny-random-clip-tf"""     ,						"""hf-internal-testing/tiny-random-bert"""	)
             _a    : Any     					=1_3
             _a    : List[Any]     					=floats_tensor(
                 [
                     batch_size,
                     model.vision_model.config.num_channels,
                     model.vision_model.config.image_size,
                     model.vision_model.config.image_size,
                 ]	)
             _a    : Any     					=ids_tensor([batch_size, 4]     ,						model.text_model.config.vocab_size	)
             _a    : List[Any]     					=random_attention_mask([batch_size, 4]	)
             _a    : List[Any]     					={'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

             return model, inputs






      def __UpperCAmelCase       ( self			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Any	)       ->					List[Any]:




             '''simple docstring'''
             _a    : Optional[int]     					=TFCLIPVisionModel(SCREAMING_SNAKE_CASE     ,						name="""vision_model"""	)
             _a    : List[str]     					=TFBertModel(SCREAMING_SNAKE_CASE     ,						name="""text_model"""	)
             return vision_model, text_model






      def __UpperCAmelCase       ( self			:str	)       ->					Tuple:




             '''simple docstring'''
             _a    : Union[str, Any]     					=TFCLIPVisionModelTester(self	)
             _a    : Optional[Any]     					=TFBertModelTester(self	)
             _a    : str     					=clip_model_tester.prepare_config_and_inputs()
             _a    : Optional[int]     					=bert_model_tester.prepare_config_and_inputs()

             _a    : List[Any]     					=vision_config_and_inputs

             (
                 _a 
             )    : List[str]     					=text_config_and_inputs

             return {
                 "text_config": text_config,
                 "vision_config": vision_config,
                 "pixel_values": pixel_values,
                 "attention_mask": input_mask,
                 "input_ids": input_ids,
                 "text_token_type_ids": token_type_ids,
                 "text_sequence_labels": sequence_labels,
                 "text_token_labels": token_labels,
                 "text_choice_labels": choice_labels,
             }







@require_vision
@require_tf
class A__       (						unittest.TestCase      ):






      @slow
      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : List[str]     					=TFVisionTextDualEncoderModel.from_pretrained(
                 """clip-italian/clip-italian"""     ,						logit_scale_init_value=1.0     ,						from_pt=SCREAMING_SNAKE_CASE	)
             _a    : str     					=VisionTextDualEncoderProcessor.from_pretrained("""clip-italian/clip-italian"""	)

             _a    : Optional[Any]     					=Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""	)
             _a    : List[Any]     					=processor(
                 text=["""una foto di un gatto""", """una foto di un cane"""]     ,						images=SCREAMING_SNAKE_CASE     ,						padding=SCREAMING_SNAKE_CASE     ,						return_tensors="""np"""	)

             _a    : List[Any]     					=model(**SCREAMING_SNAKE_CASE	)

             # verify the logits
             self.assertEqual(outputs.logits_per_image.shape     ,						(inputs.pixel_values.shape[0], inputs.input_ids.shape[0])	)
             self.assertEqual(
                 outputs.logits_per_text.shape     ,						(inputs.input_ids.shape[0], inputs.pixel_values.shape[0])     ,						)

             _a    : int     					=np.array([[1.2_284_727, 0.3_104_122]]	)

             self.assertTrue(np.allclose(outputs.logits_per_image.numpy()     ,						SCREAMING_SNAKE_CASE     ,						atol=1e-3	)	)








"""simple docstring"""






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowercase__     						=						"""\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowercase__     						=						"""\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowercase__     						=						"""
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""







def      _snake_case			(				lowercase__							,    lowercase__		):
				return float((preds == labels).mean()		)







def      _snake_case			(				lowercase__							,    lowercase__							,    lowercase__="binary"		):
				_lowerCamelCase     :			str    =    simple_accuracy(lowercase__							,    lowercase__		)
				_lowerCamelCase     :			Any    =    float(fa_score(y_true=lowercase__							,    y_pred=lowercase__							,    average=lowercase__		)		)
				return {
				    "accuracy": acc,
				    "f1": fa,
				}







def      _snake_case			(				lowercase__							,    lowercase__		):
				_lowerCamelCase     :			Any    =    {}
				for id_pred, label in zip(lowercase__							,    lowercase__		):
								_lowerCamelCase     :			Tuple    =    f'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
								_lowerCamelCase     :			Union[str, Any]    =    id_pred['prediction']
								if question_id in question_map:
												question_map[question_id].append((pred, label)		)
								else:
												_lowerCamelCase     :			Optional[Any]    =    [(pred, label)]
				_lowerCamelCase, _lowerCamelCase     :			Optional[int]    =    [], []
				for question, preds_labels in question_map.items():
								_lowerCamelCase, _lowerCamelCase     :			Tuple    =    zip(*lowercase__		)
								_lowerCamelCase     :			List[str]    =    fa_score(y_true=lowercase__							,    y_pred=lowercase__							,    average='macro'		)
								fas.append(lowercase__		)
								_lowerCamelCase     :			int    =    int(sum(pred == label for pred, label in preds_labels		) == len(lowercase__		)		)
								ems.append(lowercase__		)
				_lowerCamelCase     :			Optional[Any]    =    float(sum(lowercase__		) / len(lowercase__		)		)
				_lowerCamelCase     :			Optional[int]    =    sum(lowercase__		) / len(lowercase__		)
				_lowerCamelCase     :			List[Any]    =    float(fa_score(y_true=lowercase__							,    y_pred=[id_pred['prediction'] for id_pred in ids_preds]		)		)
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION,	_KWARGS_DESCRIPTION						)
class lowerCAmelCase__	(     datasets.Metric						):






			'''simple docstring'''



			def 					A_    (  self  ):
							if self.config_name not in [
							    "boolq",
							    "cb",
							    "copa",
							    "multirc",
							    "record",
							    "rte",
							    "wic",
							    "wsc",
							    "wsc.fixed",
							    "axb",
							    "axg",
							]:
											raise KeyError(
											    'You should supply a configuration name selected in '
											    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )
							return datasets.MetricInfo(
							    description=_DESCRIPTION  ,      citation=_CITATION  ,      inputs_description=_KWARGS_DESCRIPTION  ,      features=datasets.Features(self._get_feature_types()  )  ,      codebase_urls=[]  ,      reference_urls=[]  ,      format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None  ,      )
			def 					A_    (  self  ):
							if self.config_name == "record":
											return {
											    "predictions": {
											        "idx": {
											            "passage": datasets.Value('int64'  ),
											            "query": datasets.Value('int64'  ),
											        },
											        "prediction_text": datasets.Value('string'  ),
											    },
											    "references": {
											        "idx": {
											            "passage": datasets.Value('int64'  ),
											            "query": datasets.Value('int64'  ),
											        },
											        "answers": datasets.Sequence(datasets.Value('string'  )  ),
											    },
											}
							elif self.config_name == "multirc":
											return {
											    "predictions": {
											        "idx": {
											            "answer": datasets.Value('int64'  ),
											            "paragraph": datasets.Value('int64'  ),
											            "question": datasets.Value('int64'  ),
											        },
											        "prediction": datasets.Value('int64'  ),
											    },
											    "references": datasets.Value('int64'  ),
											}
							else:
											return {
											    "predictions": datasets.Value('int64'  ),
											    "references": datasets.Value('int64'  ),
											}






			def 					A_    (  self  ,      lowercase  ,      lowercase  ):
							if self.config_name == "axb":
											return {"matthews_correlation": matthews_corrcoef(lowercase  ,      lowercase  )}
							elif self.config_name == "cb":
											return acc_and_fa(lowercase  ,      lowercase  ,      fa_avg='macro'  )
							elif self.config_name == "record":
											_lowerCamelCase     :			List[str]    =    [
											    {
											        'qas': [
											            {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
											            for ref in references
											        ]
											    }
											]
											_lowerCamelCase     :			Union[str, Any]    =    {pred['idx']['query']: pred['prediction_text'] for pred in predictions}
											return evaluate_record(lowercase  ,      lowercase  )[0]
							elif self.config_name == "multirc":
											return evaluate_multirc(lowercase  ,      lowercase  )
							elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
											return {"accuracy": simple_accuracy(lowercase  ,      lowercase  )}
							else:
											raise KeyError(
											    'You should supply a configuration name selected in '
											    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )