


'''simple docstring'''






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


a_    :			Any      				=			"\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

a_    :			List[Any]      				=			"\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

a_    :			Optional[Any]      				=			"\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"






def 		_A (lowerCAmelCase__     :int ,  lowerCAmelCase__     :str						)      ->     str:




     '''simple docstring'''



     return float((preds == labels).mean()						)






def 		_A (lowerCAmelCase__     :Tuple ,  lowerCAmelCase__     :Dict ,  lowerCAmelCase__     :str="binary"						)      ->     List[str]:




     '''simple docstring'''



     _a						=							simple_accuracy(__SCREAMING_SNAKE_CASE ,  __SCREAMING_SNAKE_CASE						)
     _a						=							float(fa_score(y_true=__SCREAMING_SNAKE_CASE ,  y_pred=__SCREAMING_SNAKE_CASE ,  average=__SCREAMING_SNAKE_CASE						)						)
     return {
         "accuracy": acc,
         "f1": fa,
     }






def 		_A (lowerCAmelCase__     :Optional[Any] ,  lowerCAmelCase__     :Optional[int]						)      ->     List[str]:




     '''simple docstring'''



     _a						=							{}
     for id_pred, label in zip(__SCREAMING_SNAKE_CASE ,  __SCREAMING_SNAKE_CASE						):
          _a						=							f'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
          _a						=							id_pred['''prediction''']
          if question_id in question_map:
               question_map[question_id].append((pred, label)						)
          else:
               _a						=							[(pred, label)]
     _a						=							[], []
     for question, preds_labels in question_map.items():
          _a						=							zip(*__SCREAMING_SNAKE_CASE						)
          _a						=							fa_score(y_true=__SCREAMING_SNAKE_CASE ,  y_pred=__SCREAMING_SNAKE_CASE ,  average='macro'						)
          fas.append(__SCREAMING_SNAKE_CASE						)
          _a						=							int(sum(pred == label for pred, label in preds_labels						) == len(__SCREAMING_SNAKE_CASE						)						)
          ems.append(__SCREAMING_SNAKE_CASE						)
     _a						=							float(sum(__SCREAMING_SNAKE_CASE						) / len(__SCREAMING_SNAKE_CASE						)						)
     _a						=							sum(__SCREAMING_SNAKE_CASE						) / len(__SCREAMING_SNAKE_CASE						)
     _a						=							float(fa_score(y_true=__SCREAMING_SNAKE_CASE ,  y_pred=[id_pred['prediction'] for id_pred in ids_preds]						)						)
     return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     , _KWARGS_DESCRIPTION				)
class        a   (   datasets.Metric				):
 def 							__UpperCAmelCase						(       self  )    ->							List[str]:





      if self.config_name not in [
          "boolq",
          "cb",
          "copa",
          "multirc",
          "record",
          "rte",
          "wic",
          "wsc",
          "wsc.fixed",
          "axb",
          "axg",
      ]:
           raise KeyError(
               'You should supply a configuration name selected in '
               '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )
      return datasets.MetricInfo(
          description=_DESCRIPTION					, citation=_CITATION					, inputs_description=_KWARGS_DESCRIPTION					, features=datasets.Features(self._get_feature_types()  )					, codebase_urls=[]					, reference_urls=[]					, format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None					, )
 def 							__UpperCAmelCase						(       self  )    ->							str:





      if self.config_name == "record":
           return {
               "predictions": {
                   "idx": {
                       "passage": datasets.Value('int64'  ),
                       "query": datasets.Value('int64'  ),
                   },
                   "prediction_text": datasets.Value('string'  ),
               },
               "references": {
                   "idx": {
                       "passage": datasets.Value('int64'  ),
                       "query": datasets.Value('int64'  ),
                   },
                   "answers": datasets.Sequence(datasets.Value('string'  )  ),
               },
           }
      elif self.config_name == "multirc":
           return {
               "predictions": {
                   "idx": {
                       "answer": datasets.Value('int64'  ),
                       "paragraph": datasets.Value('int64'  ),
                       "question": datasets.Value('int64'  ),
                   },
                   "prediction": datasets.Value('int64'  ),
               },
               "references": datasets.Value('int64'  ),
           }
      else:
           return {
               "predictions": datasets.Value('int64'  ),
               "references": datasets.Value('int64'  ),
           }




 def 							__UpperCAmelCase						(       self					, __magic_name__					, __magic_name__  )    ->							str:





      if self.config_name == "axb":
           return {"matthews_correlation": matthews_corrcoef(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )}
      elif self.config_name == "cb":
           return acc_and_fa(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, fa_avg='macro'  )
      elif self.config_name == "record":
           _a						=							[
               {
                   '''qas''': [
                       {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                       for ref in references
                   ]
               }
           ]
           _a						=							{pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
           return evaluate_record(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )[0]
      elif self.config_name == "multirc":
           return evaluate_multirc(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )
      elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
           return {"accuracy": simple_accuracy(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )}
      else:
           raise KeyError(
               'You should supply a configuration name selected in '
               '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )



'''simple docstring'''







import torch

from transformers import AutoModel






class  lowerCAmelCase__   (  torch.nn.Module     ):






					def __init__(   self				,							__SCREAMING_SNAKE_CASE="sayef/fsner-bert-base-uncased"		):


											"""simple docstring"""





											super(__SCREAMING_SNAKE_CASE				,							self		).__init__()

											lowercase_	:     Tuple													=       AutoModel.from_pretrained(__SCREAMING_SNAKE_CASE				,							return_dict=__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Optional[Any]													=       torch.nn.CosineSimilarity(3				,							1E-0_8		)
											lowercase_	:     Optional[Any]													=       torch.nn.Softmax(dim=1		)






					def 	_snake_case    (   self				,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.bert(**__SCREAMING_SNAKE_CASE		).last_hidden_state






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return token_embeddings.sum(2				,							keepdim=__SCREAMING_SNAKE_CASE		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE=1		):


											"""simple docstring"""





											return self.softmax(T * self.cos(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Optional[Any]													=       W_supports['''sizes'''].tolist()
											lowercase_	:     Dict													=       W_supports['''start_token_id'''].item()
											lowercase_	:     List[Any]													=       W_supports['''end_token_id'''].item()

											del W_supports["sizes"]
											del W_supports["start_token_id"]
											del W_supports["end_token_id"]

											lowercase_	:     List[str]													=       self.BERT(**__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Optional[int]													=       self.BERT(**__SCREAMING_SNAKE_CASE		)

											lowercase_	:     str													=       None
											lowercase_	:     Dict													=       None

											lowercase_	:     Tuple													=       W_supports['''input_ids'''] == start_token_id
											lowercase_	:     Any													=       W_supports['''input_ids'''] == end_token_id

											for i, size in enumerate(__SCREAMING_SNAKE_CASE		):
																	if i == 0:
																							lowercase_	:     List[str]													=       0
																	else:
																							lowercase_	:     List[Any]													=       support_sizes[i - 1]

																	lowercase_	:     str													=       S[s : s + size][start_token_masks[s : s + size]]
																	lowercase_	:     Optional[int]													=       S[s : s + size][end_token_masks[s : s + size]]

																	lowercase_	:     List[str]													=       torch.matmul(q[i]				,							s_start.T		).sum(1		).softmax(0		)
																	lowercase_	:     List[str]													=       torch.matmul(q[i]				,							s_end.T		).sum(1		).softmax(0		)

																	if p_starts is not None:
																							lowercase_	:     Tuple													=       torch.vstack((p_starts, p_start)		)
																							lowercase_	:     Optional[Any]													=       torch.vstack((p_ends, p_end)		)
																	else:
																							lowercase_	:     str													=       p_start
																							lowercase_	:     int													=       p_end

											return p_starts, p_ends



