





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


snake_case_				=			"""\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

snake_case_				=			"""\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

snake_case_				=			"""
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""






def lowerCamelCase__    (							snake_case_	:  int       ,      snake_case_	:  Tuple       )      ->				Tuple:
   return float((preds == labels).mean()       )






def lowerCamelCase__    (							snake_case_	:  Any       ,      snake_case_	:  Any       ,      snake_case_	:  List[Any]="binary"       )      ->				Union[str, Any]:
   __snake_case							       =   simple_accuracy(lowerCAmelCase_       ,      lowerCAmelCase_       )
   __snake_case							       =   float(fa_score(y_true=lowerCAmelCase_       ,      y_pred=lowerCAmelCase_       ,      average=lowerCAmelCase_       )       )
   return {
       "accuracy": acc,
       "f1": fa,
   }






def lowerCamelCase__    (							snake_case_	:  Optional[int]       ,      snake_case_	:  Dict       )      ->				Tuple:
   __snake_case							       =   {}
   for id_pred, label in zip(lowerCAmelCase_       ,      lowerCAmelCase_       ):
      __snake_case							       =   f"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
      __snake_case							       =   id_pred['''prediction''']
      if question_id in question_map:
         question_map[question_id].append((pred, label)       )
      else:
         __snake_case							       =   [(pred, label)]
   __snake_case							       =   [], []
   for question, preds_labels in question_map.items():
      __snake_case							       =   zip(*lowerCAmelCase_       )
      __snake_case							       =   fa_score(y_true=lowerCAmelCase_       ,      y_pred=lowerCAmelCase_       ,      average='''macro'''       )
      fas.append(lowerCAmelCase_       )
      __snake_case							       =   int(sum(pred == label for pred, label in preds_labels       ) == len(lowerCAmelCase_       )       )
      ems.append(lowerCAmelCase_       )
   __snake_case							       =   float(sum(lowerCAmelCase_       ) / len(lowerCAmelCase_       )       )
   __snake_case							       =   sum(lowerCAmelCase_       ) / len(lowerCAmelCase_       )
   __snake_case							       =   float(fa_score(y_true=lowerCAmelCase_       ,      y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]       )       )
   return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,      _KWARGS_DESCRIPTION  )
class 		SCREAMING_SNAKE_CASE__				(       datasets.Metric  ):


      def    a (self       :      Dict      ):





         """simple docstring"""



         if self.config_name not in [
             "boolq",
             "cb",
             "copa",
             "multirc",
             "record",
             "rte",
             "wic",
             "wsc",
             "wsc.fixed",
             "axb",
             "axg",
         ]:
            raise KeyError(
                '''You should supply a configuration name selected in '''
                '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''      )
         return datasets.MetricInfo(
             description=_DESCRIPTION				,			citation=_CITATION				,			inputs_description=_KWARGS_DESCRIPTION				,			features=datasets.Features(self._get_feature_types()      )				,			codebase_urls=[]				,			reference_urls=[]				,			format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None				,			)


      def    a (self       :      Optional[Any]      ):





         """simple docstring"""



         if self.config_name == "record":
            return {
                "predictions": {
                    "idx": {
                        "passage": datasets.Value('''int64'''      ),
                        "query": datasets.Value('''int64'''      ),
                    },
                    "prediction_text": datasets.Value('''string'''      ),
                },
                "references": {
                    "idx": {
                        "passage": datasets.Value('''int64'''      ),
                        "query": datasets.Value('''int64'''      ),
                    },
                    "answers": datasets.Sequence(datasets.Value('''string'''      )      ),
                },
            }
         elif self.config_name == "multirc":
            return {
                "predictions": {
                    "idx": {
                        "answer": datasets.Value('''int64'''      ),
                        "paragraph": datasets.Value('''int64'''      ),
                        "question": datasets.Value('''int64'''      ),
                    },
                    "prediction": datasets.Value('''int64'''      ),
                },
                "references": datasets.Value('''int64'''      ),
            }
         else:
            return {
                "predictions": datasets.Value('''int64'''      ),
                "references": datasets.Value('''int64'''      ),
            }


      def    a (self       :      Optional[int]				,			a__       :      str				,			a__       :      Any      ):





         """simple docstring"""



         if self.config_name == "axb":
            return {"matthews_correlation": matthews_corrcoef(__lowercase				,			__lowercase      )}
         elif self.config_name == "cb":
            return acc_and_fa(__lowercase				,			__lowercase				,			fa_avg='''macro'''      )
         elif self.config_name == "record":
            __snake_case							       =   [
                {
                    '''qas''': [
                        {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                        for ref in references
                    ]
                }
            ]
            __snake_case							       =   {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
            return evaluate_record(__lowercase				,			__lowercase      )[0]
         elif self.config_name == "multirc":
            return evaluate_multirc(__lowercase				,			__lowercase      )
         elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
            return {"accuracy": simple_accuracy(__lowercase				,			__lowercase      )}
         else:
            raise KeyError(
                '''You should supply a configuration name selected in '''
                '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''      )





import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
_UpperCAmelCase	:     Optional[int]   ="""src/transformers"""
_UpperCAmelCase	:     str   ="""docs/source/en"""
_UpperCAmelCase	:     Optional[int]   ="""."""



def 	lowerCAmelCase       (						lowerCAmelCase_       ,      lowerCAmelCase_       ,      lowerCAmelCase_       )->       List[str]:
				with open(lowerCAmelCase_       ,      '''r'''       ,      encoding='''utf-8'''       ,      newline='''\n'''       ) as f:
								lowerCAmelCase_       :					int			 = f.readlines()
				# Find the start prompt.
				lowerCAmelCase_       :					List[Any]			 = 0
				while not lines[start_index].startswith(lowerCAmelCase_       ):
								start_index += 1
				start_index += 1

				lowerCAmelCase_       :					List[str]			 = start_index
				while not lines[end_index].startswith(lowerCAmelCase_       ):
								end_index += 1
				end_index -= 1

				while len(lines[start_index]       ) <= 1:
								start_index += 1
				while len(lines[end_index]       ) <= 1:
								end_index -= 1
				end_index += 1
				return "".join(lines[start_index:end_index]       ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
_UpperCAmelCase	:     Optional[Any]   ="""Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
_UpperCAmelCase	:     Optional[int]   =re.compile(R"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
_UpperCAmelCase	:     Dict   =re.compile(R"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
_UpperCAmelCase	:     Optional[Any]   =re.compile(R"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# This is to make sure the transformers module imported is the one in the repo.
_UpperCAmelCase	:     Optional[int]   =direct_transformers_import(TRANSFORMERS_PATH)



def 	lowerCAmelCase       (						lowerCAmelCase_       )->       Optional[int]:
				lowerCAmelCase_       :					str			 = re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''       ,      lowerCAmelCase_       )
				return [m.group(0       ) for m in matches]



def 	lowerCAmelCase       (						lowerCAmelCase_       ,      lowerCAmelCase_       )->       Optional[int]:
				lowerCAmelCase_       :					Tuple			 = 2 if text == '''✅''' or text == '''❌''' else len(lowerCAmelCase_       )
				lowerCAmelCase_       :					int			 = (width - text_length) // 2
				lowerCAmelCase_       :					Union[str, Any]			 = width - text_length - left_indent
				return " " * left_indent + text + " " * right_indent



def 	lowerCAmelCase       (						)->       str:
				lowerCAmelCase_       :					Any			 = transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
				lowerCAmelCase_       :					Dict			 = {
				    name: config_maping_names[code]
				    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
				    if code in config_maping_names
				}
				lowerCAmelCase_       :					List[Any]			 = {name: config.replace('''Config'''       ,      ''''''       ) for name, config in model_name_to_config.items()}

				# Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
				lowerCAmelCase_       :					Tuple			 = collections.defaultdict(lowerCAmelCase_       )
				lowerCAmelCase_       :					List[str]			 = collections.defaultdict(lowerCAmelCase_       )
				lowerCAmelCase_       :					Optional[Any]			 = collections.defaultdict(lowerCAmelCase_       )
				lowerCAmelCase_       :					Optional[int]			 = collections.defaultdict(lowerCAmelCase_       )
				lowerCAmelCase_       :					List[str]			 = collections.defaultdict(lowerCAmelCase_       )

				# Let's lookup through all transformers object (once).
				for attr_name in dir(lowerCAmelCase_       ):
								lowerCAmelCase_       :					Optional[int]			 = None
								if attr_name.endswith('''Tokenizer'''       ):
												lowerCAmelCase_       :					Union[str, Any]			 = slow_tokenizers
												lowerCAmelCase_       :					List[str]			 = attr_name[:-9]
								elif attr_name.endswith('''TokenizerFast'''       ):
												lowerCAmelCase_       :					int			 = fast_tokenizers
												lowerCAmelCase_       :					Union[str, Any]			 = attr_name[:-13]
								elif _re_tf_models.match(lowerCAmelCase_       ) is not None:
												lowerCAmelCase_       :					Tuple			 = tf_models
												lowerCAmelCase_       :					str			 = _re_tf_models.match(lowerCAmelCase_       ).groups()[0]
								elif _re_flax_models.match(lowerCAmelCase_       ) is not None:
												lowerCAmelCase_       :					Tuple			 = flax_models
												lowerCAmelCase_       :					Union[str, Any]			 = _re_flax_models.match(lowerCAmelCase_       ).groups()[0]
								elif _re_pt_models.match(lowerCAmelCase_       ) is not None:
												lowerCAmelCase_       :					Any			 = pt_models
												lowerCAmelCase_       :					List[Any]			 = _re_pt_models.match(lowerCAmelCase_       ).groups()[0]

								if lookup_dict is not None:
												while len(lowerCAmelCase_       ) > 0:
																if attr_name in model_name_to_prefix.values():
																				lowerCAmelCase_       :					Union[str, Any]			 = True
																				break
																# Try again after removing the last word in the name
																lowerCAmelCase_       :					Any			 = ''''''.join(camel_case_split(lowerCAmelCase_       )[:-1]       )

    # Let's build that table!
				lowerCAmelCase_       :					int			 = list(model_name_to_config.keys()       )
				model_names.sort(key=str.lower       )
				lowerCAmelCase_       :					Tuple			 = ['''Model''', '''Tokenizer slow''', '''Tokenizer fast''', '''PyTorch support''', '''TensorFlow support''', '''Flax Support''']
				# We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
				lowerCAmelCase_       :					Union[str, Any]			 = [len(lowerCAmelCase_       ) + 2 for c in columns]
				lowerCAmelCase_       :					Optional[Any]			 = max([len(lowerCAmelCase_       ) for name in model_names]       ) + 2

				# Build the table per se
				lowerCAmelCase_       :					Dict			 = '''|''' + '''|'''.join([_center_text(lowerCAmelCase_       ,      lowerCAmelCase_       ) for c, w in zip(lowerCAmelCase_       ,      lowerCAmelCase_       )]       ) + '''|\n'''
				# Use ":-----:" format to center-aligned table cell texts
				table += "|" + "|".join([''':''' + '''-''' * (w - 2) + ''':''' for w in widths]       ) + "|\n"

				lowerCAmelCase_       :					List[str]			 = {True: '''✅''', False: '''❌'''}
				for name in model_names:
								lowerCAmelCase_       :					List[Any]			 = model_name_to_prefix[name]
								lowerCAmelCase_       :					Union[str, Any]			 = [
								    name,
								    check[slow_tokenizers[prefix]],
								    check[fast_tokenizers[prefix]],
								    check[pt_models[prefix]],
								    check[tf_models[prefix]],
								    check[flax_models[prefix]],
								]
								table += "|" + "|".join([_center_text(lowerCAmelCase_       ,      lowerCAmelCase_       ) for l, w in zip(lowerCAmelCase_       ,      lowerCAmelCase_       )]       ) + "|\n"
				return table



def 	lowerCAmelCase       (						lowerCAmelCase_=False       )->       Tuple:
				lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_       :					str			 = _find_text_in_file(
				    filename=os.path.join(lowerCAmelCase_       ,      '''index.md'''       )       ,      start_prompt='''<!--This table is updated automatically from the auto modules'''       ,      end_prompt='''<!-- End table-->'''       ,      )
				lowerCAmelCase_       :					Tuple			 = get_model_table_from_auto_modules()

				if current_table != new_table:
								if overwrite:
												with open(os.path.join(lowerCAmelCase_       ,      '''index.md'''       )       ,      '''w'''       ,      encoding='''utf-8'''       ,      newline='''\n'''       ) as f:
																f.writelines(lines[:start_index] + [new_table] + lines[end_index:]       )
								else:
												raise ValueError(
												    '''The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this.'''       )


if __name__ == "__main__":
						_UpperCAmelCase	:     List[Any]   =argparse.ArgumentParser()
						parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
						_UpperCAmelCase	:     Tuple   =parser.parse_args()

						check_model_table(args.fix_and_overwrite)