'''simple docstring'''

import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm
  import torch_xla.debug.metrics as met


class 					_lowercase						(   lowerCAmelCase ):
      '''simple docstring'''


      def __init__(					self       ,*lowerCamelCase_       ,lowerCamelCase_=None       ,lowerCamelCase_=None       ,**lowerCamelCase_				)      ->      int:



         '''simple docstring'''



         super().__init__(*lowerCamelCase_       ,**lowerCamelCase_				)
         UpperCAmelCase__				:					Tuple 					=	eval_examples
         UpperCAmelCase__				:					Optional[Any] 					=	post_process_function




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_ = None       ,lowerCamelCase_=None       ,lowerCamelCase_ = None       ,lowerCamelCase_ = "eval"       ,**lowerCamelCase_       ,)      ->      Dict[str, float]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	gen_kwargs.copy()
         UpperCAmelCase__				:					str 					=	(
             gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length'''				) is not None else self.args.generation_max_length
         )
         UpperCAmelCase__				:					Union[str, Any] 					=	(
             gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''				) is not None else self.args.generation_num_beams
         )
         UpperCAmelCase__				:					Union[str, Any] 					=	gen_kwargs

         UpperCAmelCase__				:					int 					=	self.eval_dataset if eval_dataset is None else eval_dataset
         UpperCAmelCase__				:					str 					=	self.get_eval_dataloader(lowerCamelCase_				)
         UpperCAmelCase__				:					Optional[Any] 					=	self.eval_examples if eval_examples is None else eval_examples

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase__				:					Optional[int] 					=	self.compute_metrics
         UpperCAmelCase__				:					Tuple 					=	None
         UpperCAmelCase__				:					Optional[int] 					=	time.time()
         UpperCAmelCase__				:					int 					=	self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         try:
            UpperCAmelCase__				:					List[str] 					=	eval_loop(
                lowerCamelCase_       ,description='''Evaluation'''       ,prediction_loss_only=True if compute_metrics is None else None       ,ignore_keys=lowerCamelCase_       ,metric_key_prefix=lowerCamelCase_       ,)
         finally:
            UpperCAmelCase__				:					Optional[Any] 					=	compute_metrics
         UpperCAmelCase__				:					List[str] 					=	self.args.eval_batch_size * self.args.world_size
         if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
            start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
         output.metrics.update(
             speed_metrics(
                 lowerCamelCase_       ,lowerCamelCase_       ,num_samples=output.num_samples       ,num_steps=math.ceil(output.num_samples / total_batch_size				)       ,)				)

         if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
            # Only the main node write the results by default
            UpperCAmelCase__				:					Optional[Any] 					=	self.post_process_function(lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)
            UpperCAmelCase__				:					Union[str, Any] 					=	self.compute_metrics(lowerCamelCase_				)

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()				):
               if not key.startswith(f'''{metric_key_prefix}_'''				):
                  UpperCAmelCase__				:					List[str] 					=	metrics.pop(lowerCamelCase_				)

            metrics.update(output.metrics				)
         else:
            UpperCAmelCase__				:					List[Any] 					=	output.metrics

         if self.args.should_log:
            # Only the main node log the results by default
            self.log(lowerCamelCase_				)

         if self.args.tpu_metrics_debug or self.args.debug:
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
            xm.master_print(met.metrics_report()				)

         UpperCAmelCase__				:					Union[str, Any] 					=	self.callback_handler.on_evaluate(self.args       ,self.state       ,self.control       ,lowerCamelCase_				)
         return metrics




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_=None       ,lowerCamelCase_ = "test"       ,**lowerCamelCase_				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	gen_kwargs.copy()

         UpperCAmelCase__				:					Union[str, Any] 					=	self.get_test_dataloader(lowerCamelCase_				)

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase__				:					List[Any] 					=	self.compute_metrics
         UpperCAmelCase__				:					int 					=	None
         UpperCAmelCase__				:					List[str] 					=	time.time()
         UpperCAmelCase__				:					Optional[Any] 					=	self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         try:
            UpperCAmelCase__				:					str 					=	eval_loop(
                lowerCamelCase_       ,description='''Prediction'''       ,prediction_loss_only=True if compute_metrics is None else None       ,ignore_keys=lowerCamelCase_       ,metric_key_prefix=lowerCamelCase_       ,)
         finally:
            UpperCAmelCase__				:					str 					=	compute_metrics

         UpperCAmelCase__				:					List[str] 					=	self.args.eval_batch_size * self.args.world_size
         if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
            start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
         output.metrics.update(
             speed_metrics(
                 lowerCamelCase_       ,lowerCamelCase_       ,num_samples=output.num_samples       ,num_steps=math.ceil(output.num_samples / total_batch_size				)       ,)				)
         if self.post_process_function is None or self.compute_metrics is None:
            return output

         UpperCAmelCase__				:					Optional[int] 					=	self.post_process_function(lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,'''predict'''				)
         UpperCAmelCase__				:					str 					=	self.compute_metrics(lowerCamelCase_				)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()				):
            if not key.startswith(f'''{metric_key_prefix}_'''				):
               UpperCAmelCase__				:					Any 					=	metrics.pop(lowerCamelCase_				)
         metrics.update(output.metrics				)
         return PredictionOutput(predictions=predictions.predictions       ,label_ids=predictions.label_ids       ,metrics=lowerCamelCase_				)





'''simple docstring'''

from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Sequence, Value
from .base import TaskTemplate


@dataclass(frozen=lowerCAmelCase )
class 					_lowercase						(   lowerCAmelCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   str   				= field(default='''question-answering-extractive'''		,metadata={'''include_in_asdict_even_if_is_default''': True} )
      UpperCAmelCase_   :   ClassVar[Features]   				= Features({'''question''': Value('''string''' ), '''context''': Value('''string''' )} )
      UpperCAmelCase_   :   ClassVar[Features]   				= Features(
          {
              '''answers''': Sequence(
                  {
                      '''text''': Value('''string''' ),
                      '''answer_start''': Value('''int32''' ),
                  } )
          } )
      UpperCAmelCase_   :   str   				= "question"
      UpperCAmelCase_   :   str   				= "context"
      UpperCAmelCase_   :   str   				= "answers"




      @property
      def 		lowerCAmelCase__       (					self				)      ->      Dict[str, str]:



         '''simple docstring'''



         return {self.question_column: "question", self.context_column: "context", self.answers_column: "answers"}




