
'''simple docstring'''







from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_lowercase :  Any    							=       "\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

_lowercase :  List[Any]    							=       "\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

_lowercase :  Optional[Any]    							=       "\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: int      ,      __SCREAMING_SNAKE_CASE					: str							):



						"""simple docstring"""





						return float((preds == labels).mean()							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Tuple      ,      __SCREAMING_SNAKE_CASE					: Dict      ,      __SCREAMING_SNAKE_CASE					: str="binary"							):



						"""simple docstring"""





						lowercase_	:     Dict													=       simple_accuracy(__SCREAMING_SNAKE_CASE      ,      __SCREAMING_SNAKE_CASE							)
						lowercase_	:     Tuple													=       float(fa_score(y_true=__SCREAMING_SNAKE_CASE      ,      y_pred=__SCREAMING_SNAKE_CASE      ,      average=__SCREAMING_SNAKE_CASE							)							)
						return {
						    "accuracy": acc,
						    "f1": fa,
						}
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Optional[Any]      ,      __SCREAMING_SNAKE_CASE					: Optional[int]							):



						"""simple docstring"""





						lowercase_	:     str													=       {}
						for id_pred, label in zip(__SCREAMING_SNAKE_CASE      ,      __SCREAMING_SNAKE_CASE							):
												lowercase_	:     str													=       F'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
												lowercase_	:     Any													=       id_pred['''prediction''']
												if question_id in question_map:
																		question_map[question_id].append((pred, label)							)
												else:
																		lowercase_	:     List[str]													=       [(pred, label)]
						lowercase_						, lowercase_	:     Dict													=       [], []
						for question, preds_labels in question_map.items():
												lowercase_						, lowercase_	:     List[Any]													=       zip(*__SCREAMING_SNAKE_CASE							)
												lowercase_	:     Union[str, Any]													=       fa_score(y_true=__SCREAMING_SNAKE_CASE      ,      y_pred=__SCREAMING_SNAKE_CASE      ,      average='''macro'''							)
												fas.append(__SCREAMING_SNAKE_CASE							)
												lowercase_	:     Optional[Any]													=       int(sum(pred == label for pred, label in preds_labels							) == len(__SCREAMING_SNAKE_CASE							)							)
												ems.append(__SCREAMING_SNAKE_CASE							)
						lowercase_	:     str													=       float(sum(__SCREAMING_SNAKE_CASE							) / len(__SCREAMING_SNAKE_CASE							)							)
						lowercase_	:     Tuple													=       sum(__SCREAMING_SNAKE_CASE							) / len(__SCREAMING_SNAKE_CASE							)
						lowercase_	:     Optional[Any]													=       float(fa_score(y_true=__SCREAMING_SNAKE_CASE      ,      y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]							)							)
						return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,						_KWARGS_DESCRIPTION     )
class  lowerCAmelCase__   (  datasets.Metric     ):






					def 	_snake_case    (   self		):


											"""simple docstring"""





											if self.config_name not in [
											    "boolq",
											    "cb",
											    "copa",
											    "multirc",
											    "record",
											    "rte",
											    "wic",
											    "wsc",
											    "wsc.fixed",
											    "axb",
											    "axg",
											]:
																	raise KeyError(
																	    '''You should supply a configuration name selected in '''
																	    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''		)
											return datasets.MetricInfo(
											    description=_DESCRIPTION				,							citation=_CITATION				,							inputs_description=_KWARGS_DESCRIPTION				,							features=datasets.Features(self._get_feature_types()		)				,							codebase_urls=[]				,							reference_urls=[]				,							format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None				,							)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											if self.config_name == "record":
																	return {
																	    "predictions": {
																	        "idx": {
																	            "passage": datasets.Value('''int64'''		),
																	            "query": datasets.Value('''int64'''		),
																	        },
																	        "prediction_text": datasets.Value('''string'''		),
																	    },
																	    "references": {
																	        "idx": {
																	            "passage": datasets.Value('''int64'''		),
																	            "query": datasets.Value('''int64'''		),
																	        },
																	        "answers": datasets.Sequence(datasets.Value('''string'''		)		),
																	    },
																	}
											elif self.config_name == "multirc":
																	return {
																	    "predictions": {
																	        "idx": {
																	            "answer": datasets.Value('''int64'''		),
																	            "paragraph": datasets.Value('''int64'''		),
																	            "question": datasets.Value('''int64'''		),
																	        },
																	        "prediction": datasets.Value('''int64'''		),
																	    },
																	    "references": datasets.Value('''int64'''		),
																	}
											else:
																	return {
																	    "predictions": datasets.Value('''int64'''		),
																	    "references": datasets.Value('''int64'''		),
																	}






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if self.config_name == "axb":
																	return {"matthews_correlation": matthews_corrcoef(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)}
											elif self.config_name == "cb":
																	return acc_and_fa(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							fa_avg='''macro'''		)
											elif self.config_name == "record":
																	lowercase_	:     List[str]													=       [
																	    {
																	        '''qas''': [
																	            {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
																	            for ref in references
																	        ]
																	    }
																	]
																	lowercase_	:     str													=       {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
																	return evaluate_record(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)[0]
											elif self.config_name == "multirc":
																	return evaluate_multirc(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)
											elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																	return {"accuracy": simple_accuracy(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)}
											else:
																	raise KeyError(
																	    '''You should supply a configuration name selected in '''
																	    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''		)





'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_lowercase :  Union[str, Any]    							=       {
    "configuration_pix2struct": [
        "PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "Pix2StructConfig",
        "Pix2StructTextConfig",
        "Pix2StructVisionConfig",
    ],
    "processing_pix2struct": ["Pix2StructProcessor"],
}

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							_lowercase :  Dict    							=       ["Pix2StructImageProcessor"]


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							_lowercase :  List[str]    							=       [
							    "PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "Pix2StructPreTrainedModel",
							    "Pix2StructForConditionalGeneration",
							    "Pix2StructVisionModel",
							    "Pix2StructTextModel",
							]

if TYPE_CHECKING:
							from .configuration_pixastruct import (
							    PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    PixaStructConfig,
							    PixaStructTextConfig,
							    PixaStructVisionConfig,
							)
							from .processing_pixastruct import PixaStructProcessor

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .image_processing_pixastruct import PixaStructImageProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_pixastruct import (
														    PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    PixaStructForConditionalGeneration,
														    PixaStructPreTrainedModel,
														    PixaStructTextModel,
														    PixaStructVisionModel,
														)

else:
							import sys

							_lowercase :  Optional[int]    							=       _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



