




import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only
from utils_rag import save_json







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Union[str, Any]     ):
					__lowerCAmelCase				   =       filter(lambda SCREAMING_SNAKE_CASE_     : p.requires_grad      ,      model.parameters()     )
					__lowerCAmelCase				   =       sum([np.prod(p.size()     ) for p in model_parameters]     )
					return params


UpperCamelCase__    			=			logging.getLogger(__name__)







def 	_a      (						SCREAMING_SNAKE_CASE_							:				List[Any]      ,      SCREAMING_SNAKE_CASE_							:				Any     ):
					if metric == "rouge2":
										__lowerCAmelCase				   =       "{val_avg_rouge2:.4f}-{step_count}"
					elif metric == "bleu":
										__lowerCAmelCase				   =       "{val_avg_bleu:.4f}-{step_count}"
					elif metric == "em":
										__lowerCAmelCase				   =       "{val_avg_em:.4f}-{step_count}"
					elif metric == "loss":
										__lowerCAmelCase				   =       "{val_avg_loss:.4f}-{step_count}"
					else:
										raise NotImplementedError(
										    F"""seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this"""
										    " function."     )

					__lowerCAmelCase				   =       ModelCheckpoint(
					    dirpath=SCREAMING_SNAKE_CASE_      ,      filename=SCREAMING_SNAKE_CASE_      ,      monitor=F"""val_{metric}"""      ,      mode="max"      ,      save_top_k=1      ,      every_n_epochs=1      ,      )

					return checkpoint_callback







def 	_a      (						SCREAMING_SNAKE_CASE_							:				List[str]      ,      SCREAMING_SNAKE_CASE_							:				str     ):
					return EarlyStopping(
					    monitor=F"""val_{metric}"""      ,      mode="min" if "loss" in metric else "max"      ,      patience=SCREAMING_SNAKE_CASE_      ,      verbose=SCREAMING_SNAKE_CASE_      ,      )


class 			a__    (					pl.Callback      ):






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       {f"""lr_group_{i}""": param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups			)}
											pl_module.logger.log_metrics(_A			)






						@rank_zero_only
						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A		,			_A		,			_A=True			):




											"""simple docstring"""



											logger.info(f"""***** {type_path} results at step {trainer.global_step:05d} *****"""			)
											__lowerCAmelCase				   =       trainer.callback_metrics
											trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]}			)
											# Log results
											__lowerCAmelCase				   =       Path(pl_module.hparams.output_dir			)
											if type_path == "test":
																__lowerCAmelCase				   =       od / "test_results.txt"
																__lowerCAmelCase				   =       od / "test_generations.txt"
											else:
																# this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
																# If people want this it will be easy enough to add back.
																__lowerCAmelCase				   =       od / f"""{type_path}_results/{trainer.global_step:05d}.txt"""
																__lowerCAmelCase				   =       od / f"""{type_path}_generations/{trainer.global_step:05d}.txt"""
																results_file.parent.mkdir(exist_ok=_A			)
																generations_file.parent.mkdir(exist_ok=_A			)
											with open(_A		,			"a+"			) as writer:
																for key in sorted(_A			):
																					if key in ["log", "progress_bar", "preds"]:
																										continue
																					__lowerCAmelCase				   =       metrics[key]
																					if isinstance(_A		,			torch.Tensor			):
																										__lowerCAmelCase				   =       val.item()
																					__lowerCAmelCase				   =       f"""{key}: {val:.6f}\n"""
																					writer.write(_A			)

											if not save_generations:
																return

											if "preds" in metrics:
																__lowerCAmelCase				   =       "\n".join(metrics["preds"]			)
																generations_file.open("w+"			).write(_A			)






						@rank_zero_only
						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A			):




											"""simple docstring"""



											try:
																__lowerCAmelCase				   =       pl_module.model.model.num_parameters()
											except AttributeError:
																__lowerCAmelCase				   =       pl_module.model.num_parameters()

											__lowerCAmelCase				   =       count_trainable_parameters(_A			)
											# mp stands for million parameters
											trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1E6, "grad_mp": n_trainable_pars / 1E6}			)






						@rank_zero_only
						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A			):




											"""simple docstring"""



											save_json(pl_module.metrics		,			pl_module.metrics_save_path			)
											return self._write_logs(_A		,			_A		,			"test"			)






						@rank_zero_only
						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A			):




											"""simple docstring"""



											save_json(pl_module.metrics		,			pl_module.metrics_save_path			)
											# Uncommenting this will save val generations
											# return self._write_logs(trainer, pl_module, "valid")










import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
					import torch_xla.core.xla_model as xm
					import torch_xla.debug.metrics as met
class 			a__    (					snake_case__      ):






						def __init__(   self		,			*_A		,			_A=None		,			_A=None		,			**_A			):




											"""simple docstring"""



											super().__init__(*_A		,			**_A			)
											__lowerCAmelCase				   =       eval_examples
											__lowerCAmelCase				   =       post_process_function






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A = None		,			_A=None		,			_A = None		,			_A = "eval"		,			**_A		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()
											__lowerCAmelCase				   =       (
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"			) is not None else self.args.generation_max_length
											)
											__lowerCAmelCase				   =       (
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"			) is not None else self.args.generation_num_beams
											)
											__lowerCAmelCase				   =       gen_kwargs

											__lowerCAmelCase				   =       self.eval_dataset if eval_dataset is None else eval_dataset
											__lowerCAmelCase				   =       self.get_eval_dataloader(_A			)
											__lowerCAmelCase				   =       self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Evaluation"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics
											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A			)
																__lowerCAmelCase				   =       self.compute_metrics(_A			)

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()			):
																					if not key.startswith(f"""{metric_key_prefix}_"""			):
																										__lowerCAmelCase				   =       metrics.pop(_A			)

																metrics.update(output.metrics			)
											else:
																__lowerCAmelCase				   =       output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(_A			)

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()			)

											__lowerCAmelCase				   =       self.callback_handler.on_evaluate(self.args		,			self.state		,			self.control		,			_A			)
											return metrics






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A		,			_A=None		,			_A = "test"		,			**_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()

											__lowerCAmelCase				   =       self.get_test_dataloader(_A			)

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Prediction"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics

											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A		,			"predict"			)
											__lowerCAmelCase				   =       self.compute_metrics(_A			)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()			):
																if not key.startswith(f"""{metric_key_prefix}_"""			):
																					__lowerCAmelCase				   =       metrics.pop(_A			)
											metrics.update(output.metrics			)
											return PredictionOutput(predictions=predictions.predictions		,			label_ids=predictions.label_ids		,			metrics=_A			)




