





import re
import string

import numpy as np

import datasets


__A									=				"\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n"

__A									=				"\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results[\"exact_match\"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results[\"exact_match\"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\", \"YELL\"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results[\"exact_match\"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\", \"YELL\"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results[\"exact_match\"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"The cat sat on the mat.\", \"Theaters are great.\", \"It's like comparing oranges and apples.\"]\n    >>> preds = [\"The cat sat on the mat?\", \"Theaters are great.\", \"It's like comparing apples and oranges.\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results[\"exact_match\"], 1))\n    33.3\n\n"

__A									=				"\n"



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,							_KWARGS_DESCRIPTION    )
class    _SCREAMING_SNAKE_CASE						(	datasets.Metric    ):







 '''simple docstring'''

 def 							SCREAMING_SNAKE_CASE_					(self				:   str)       ->List[Any]:



   '''simple docstring'''



   return datasets.MetricInfo(
       description=_DESCRIPTION       , citation=_CITATION       , inputs_description=_KWARGS_DESCRIPTION       , features=datasets.Features(
           {
               "predictions": datasets.Value("string"       , id="sequence"),
               "references": datasets.Value("string"       , id="sequence"),
           })       , reference_urls=[]       , )







 def 							SCREAMING_SNAKE_CASE_					(self				:   Any       , UpperCAmelCase_				:   str       , UpperCAmelCase_				:   Optional[Any]       , UpperCAmelCase_				:   Optional[Any]=None       , UpperCAmelCase_				:   Union[str, Any]=False       , UpperCAmelCase_				:   Optional[Any]=False       , UpperCAmelCase_				:   int=False       , )       ->Union[str, Any]:



   '''simple docstring'''



   if regexes_to_ignore is not None:
     for s in regexes_to_ignore:
       lowerCamelCase__:      List[str]			  =np.array([re.sub(UpperCAmelCase_       , ""       , UpperCAmelCase_) for x in predictions])
       lowerCamelCase__:      Tuple			  =np.array([re.sub(UpperCAmelCase_       , ""       , UpperCAmelCase_) for x in references])
   else:
     lowerCamelCase__:      Union[str, Any]			  =np.asarray(UpperCAmelCase_)
     lowerCamelCase__:      int			  =np.asarray(UpperCAmelCase_)

   if ignore_case:
     lowerCamelCase__:      Union[str, Any]			  =np.char.lower(UpperCAmelCase_)
     lowerCamelCase__:      Dict			  =np.char.lower(UpperCAmelCase_)

   if ignore_punctuation:
     lowerCamelCase__:      Any			  =string.punctuation.maketrans(""       , ""       , string.punctuation)
     lowerCamelCase__:      Any			  =np.char.translate(UpperCAmelCase_       , table=UpperCAmelCase_)
     lowerCamelCase__:      Tuple			  =np.char.translate(UpperCAmelCase_       , table=UpperCAmelCase_)

   if ignore_numbers:
     lowerCamelCase__:      int			  =string.digits.maketrans(""       , ""       , string.digits)
     lowerCamelCase__:      Optional[Any]			  =np.char.translate(UpperCAmelCase_       , table=UpperCAmelCase_)
     lowerCamelCase__:      Dict			  =np.char.translate(UpperCAmelCase_       , table=UpperCAmelCase_)

   lowerCamelCase__:      str			  =predictions == references

   return {"exact_match": np.mean(UpperCAmelCase_) * 100}







import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
					import torch_xla.core.xla_model as xm
					import torch_xla.debug.metrics as met
class 			a__    (					snake_case__      ):






						def __init__(   self		,			*_A		,			_A=None		,			_A=None		,			**_A			):




											"""simple docstring"""



											super().__init__(*_A		,			**_A			)
											__lowerCAmelCase				   =       eval_examples
											__lowerCAmelCase				   =       post_process_function






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A = None		,			_A=None		,			_A = None		,			_A = "eval"		,			**_A		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()
											__lowerCAmelCase				   =       (
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"			) is not None else self.args.generation_max_length
											)
											__lowerCAmelCase				   =       (
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"			) is not None else self.args.generation_num_beams
											)
											__lowerCAmelCase				   =       gen_kwargs

											__lowerCAmelCase				   =       self.eval_dataset if eval_dataset is None else eval_dataset
											__lowerCAmelCase				   =       self.get_eval_dataloader(_A			)
											__lowerCAmelCase				   =       self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Evaluation"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics
											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A			)
																__lowerCAmelCase				   =       self.compute_metrics(_A			)

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()			):
																					if not key.startswith(f"""{metric_key_prefix}_"""			):
																										__lowerCAmelCase				   =       metrics.pop(_A			)

																metrics.update(output.metrics			)
											else:
																__lowerCAmelCase				   =       output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(_A			)

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()			)

											__lowerCAmelCase				   =       self.callback_handler.on_evaluate(self.args		,			self.state		,			self.control		,			_A			)
											return metrics






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A		,			_A=None		,			_A = "test"		,			**_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()

											__lowerCAmelCase				   =       self.get_test_dataloader(_A			)

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Prediction"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics

											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A		,			"predict"			)
											__lowerCAmelCase				   =       self.compute_metrics(_A			)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()			):
																if not key.startswith(f"""{metric_key_prefix}_"""			):
																					__lowerCAmelCase				   =       metrics.pop(_A			)
											metrics.update(output.metrics			)
											return PredictionOutput(predictions=predictions.predictions		,			label_ids=predictions.label_ids		,			metrics=_A			)




