



"""simple docstring"""

import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


lowercase__    :  int							    =	logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
     import torch_xla.core.xla_model as xm
     import torch_xla.debug.metrics as met




class        _UpperCAmelCase       (			lowerCAmelCase__):

 def __init__(     self		:       str  ,     *lowercase_		:       Tuple  ,     lowercase_		:       str=None  ,     lowercase_		:       Optional[Any]=None  ,     lowercase_		:       Optional[Any]=None  ,     **lowercase_		:       Optional[int]				):
     super().__init__(*lowercase_  ,     **lowercase_				)
     snake_case_ :     Tuple				=	eval_examples
     snake_case_ :     List[str]				=	post_process_function
     snake_case_ :     List[str]				=	quant_trainer_args
     snake_case_ :     List[str]				=	128  # default number of calibration samples

 def 						_snake_case					(     self		:       Tuple  ,     lowercase_		:       List[Any]=None				):
     if calib_dataset is None and self.calib_dataset is None:
         raise ValueError('''Trainer: calibration requires an calib_dataset.'''				)
     snake_case_ :     Union[str, Any]				=	calib_dataset if calib_dataset is not None else self.calib_dataset

     snake_case_ :     Optional[Any]				=	self._remove_unused_columns(lowercase_  ,     description='''Calibration'''				)

     return DataLoader(
         lowercase_  ,     batch_size=self.args.eval_batch_size  ,     collate_fn=self.data_collator  ,     drop_last=self.args.dataloader_drop_last  ,     num_workers=self.args.dataloader_num_workers  ,     pin_memory=self.args.dataloader_pin_memory  ,     shuffle=lowercase_  ,     )

 def 						_snake_case					(     self		:       Union[str, Any]  ,     lowercase_		:       Tuple=None				):
     snake_case_ :     Any				=	self.train_dataset if calib_dataset is None else calib_dataset
     snake_case_ :     Any				=	self.get_calib_dataloader(lowercase_				)

     snake_case_ :     Any				=	self.model
     quant_trainer.configure_model(lowercase_  ,     self.quant_trainer_args  ,     calib=lowercase_				)
     model.eval()
     quant_trainer.enable_calibration(lowercase_				)

     logger.info('''***** Running calibration *****'''				)
     logger.info(f"  Num examples = {self.calib_num}"				)
     logger.info(f"  Batch size = {calib_dataloader.batch_size}"				)

     for step, inputs in enumerate(lowercase_				):
         # Prediction step
         snake_case_,	snake_case_,	snake_case_ :     List[str]				=	self.prediction_step(lowercase_  ,     lowercase_  ,     prediction_loss_only=lowercase_				)
         if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
             break

     quant_trainer.finish_calibration(lowercase_  ,     self.quant_trainer_args				)
     snake_case_ :     str				=	model

 def 						_snake_case					(     self		:       Union[str, Any]  ,     lowercase_		:       List[str]=None  ,     lowercase_		:       Union[str, Any]=None  ,     lowercase_		:       List[Any]=None  ,     lowercase_		:       str = "eval"				):
     snake_case_ :     Dict				=	self.eval_dataset if eval_dataset is None else eval_dataset
     snake_case_ :     Optional[int]				=	self.get_eval_dataloader(lowercase_				)
     snake_case_ :     Dict				=	self.eval_examples if eval_examples is None else eval_examples

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_ :     Union[str, Any]				=	self.compute_metrics
     snake_case_ :     List[Any]				=	None
     snake_case_ :     str				=	self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
         snake_case_ :     Any				=	eval_loop(
             lowercase_  ,     description='''Evaluation'''  ,     prediction_loss_only=True if compute_metrics is None else None  ,     ignore_keys=lowercase_  ,     )
     finally:
         snake_case_ :     List[Any]				=	compute_metrics

     if self.post_process_function is not None and self.compute_metrics is not None:
         snake_case_ :     Optional[int]				=	self.post_process_function(lowercase_  ,     lowercase_  ,     output.predictions				)
         snake_case_ :     Dict				=	self.compute_metrics(lowercase_				)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()				):
             if not key.startswith(f"{metric_key_prefix}_"				):
                 snake_case_ :     Tuple				=	metrics.pop(lowercase_				)

         self.log(lowercase_				)
     else:
         snake_case_ :     int				=	{}

     if self.args.tpu_metrics_debug or self.args.debug:
         # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
         xm.master_print(met.metrics_report()				)

     snake_case_ :     int				=	self.callback_handler.on_evaluate(self.args  ,     self.state  ,     self.control  ,     lowercase_				)
     return metrics

 def 						_snake_case					(     self		:       Tuple  ,     lowercase_		:       Union[str, Any]  ,     lowercase_		:       Optional[Any]  ,     lowercase_		:       Optional[int]=None  ,     lowercase_		:       str = "test"				):
     snake_case_ :     int				=	self.get_test_dataloader(lowercase_				)

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_ :     List[Any]				=	self.compute_metrics
     snake_case_ :     Dict				=	None
     snake_case_ :     int				=	self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
         snake_case_ :     int				=	eval_loop(
             lowercase_  ,     description='''Prediction'''  ,     prediction_loss_only=True if compute_metrics is None else None  ,     ignore_keys=lowercase_  ,     )
     finally:
         snake_case_ :     List[Any]				=	compute_metrics

     if self.post_process_function is None or self.compute_metrics is None:
         return output

     snake_case_ :     Tuple				=	self.post_process_function(lowercase_  ,     lowercase_  ,     output.predictions  ,     '''predict'''				)
     snake_case_ :     str				=	self.compute_metrics(lowercase_				)

     # Prefix all keys with metric_key_prefix + '_'
     for key in list(metrics.keys()				):
         if not key.startswith(f"{metric_key_prefix}_"				):
             snake_case_ :     List[str]				=	metrics.pop(lowercase_				)

     return PredictionOutput(predictions=predictions.predictions  ,     label_ids=predictions.label_ids  ,     metrics=lowercase_				)






 def 						_snake_case					(     self		:       Tuple  ,     lowercase_		:       Tuple="./"				):
     snake_case_ :     Union[str, Any]				=	self.eval_dataset
     snake_case_ :     Tuple				=	self.get_eval_dataloader(lowercase_				)

     snake_case_ :     int				=	next(iter(lowercase_				)				)

     # saving device - to make it consistent
     snake_case_ :     Optional[int]				=	torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''				)

     # convert to tuple
     snake_case_ :     str				=	tuple(v.to(lowercase_				) for k, v in batch.items()				)

     logger.info('''Converting model to be onnx compatible'''				)
     from pytorch_quantization.nn import TensorQuantizer

     snake_case_ :     Dict				=	True

     snake_case_ :     List[Any]				=	self.model.to(lowercase_				)

     model.eval()
     model.float()

     snake_case_ :     Dict				=	model.module if hasattr(lowercase_  ,     '''module'''				) else model
     quant_trainer.configure_model(lowercase_  ,     self.quant_trainer_args				)

     snake_case_ :     List[str]				=	os.path.join(lowercase_  ,     '''model.onnx'''				)
     logger.info(f"exporting model to {output_model_file}"				)

     snake_case_ :     List[str]				=	{0: '''batch_size''', 1: '''seq_len'''}

     torch.onnx.export(
         lowercase_  ,     lowercase_  ,     lowercase_  ,     export_params=lowercase_  ,     opset_version=13  ,     do_constant_folding=lowercase_  ,     input_names=['''input_ids''', '''attention_mask''', '''token_type_ids''']  ,     output_names=['''output_start_logits''', '''output_end_logits''']  ,     dynamic_axes={
             '''input_ids''': axes,
             '''attention_mask''': axes,
             '''token_type_ids''': axes,
             '''output_start_logits''': axes,
             '''output_end_logits''': axes,
         }  ,     verbose=lowercase_  ,     )
     logger.info('''onnx export finished'''				)










"""simple docstring"""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_pegasus import PegasusTokenizer
else:
     lowercase__    :  int							    =	None


lowercase__    :  Any							    =	logging.get_logger(__name__)


lowercase__    :  List[str]							    =	'''▁'''

lowercase__    :  Optional[int]							    =	{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase__    :  str							    =	{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''},
    '''tokenizer_file''': {
        '''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'''
    },
}

lowercase__    :  List[Any]							    =	{
    '''google/pegasus-xsum''': 5_12,
}




class        _UpperCAmelCase       (			lowerCAmelCase__):
 _lowerCAmelCase   :			List[str]							=		VOCAB_FILES_NAMES
 _lowerCAmelCase   :			List[str]							=		PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase   :			Any							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase   :			Tuple							=		PegasusTokenizer
 _lowerCAmelCase   :			str							=		["""input_ids""", """attention_mask"""]

 def __init__(     self		:       Any  ,     lowercase_		:       Optional[Any]=None  ,     lowercase_		:       int=None  ,     lowercase_		:       Tuple="<pad>"  ,     lowercase_		:       int="</s>"  ,     lowercase_		:       Tuple="<unk>"  ,     lowercase_		:       str="<mask_2>"  ,     lowercase_		:       Optional[Any]="<mask_1>"  ,     lowercase_		:       str=None  ,     lowercase_		:       List[str]=103  ,     **lowercase_		:       List[Any]  ,     ):
     snake_case_ :     Dict				=	offset

     if additional_special_tokens is not None:
         if not isinstance(lowercase_  ,     lowercase_				):
             raise TypeError(
                 f"additional_special_tokens should be of type {type(lowercase_				)}, but is"
                 f" {type(lowercase_				)}"				)

         snake_case_ :     str				=	(
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f"<unk_{i}>" for i in range(len(lowercase_				)  ,     self.offset - 1				)
         ]

         if len(set(lowercase_				)				) != len(lowercase_				):
             raise ValueError(
                 '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
                 f" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."				)
         snake_case_ :     Union[str, Any]				=	additional_special_tokens_extended
     else:
         snake_case_ :     Dict				=	[mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f"<unk_{i}>" for i in range(2  ,     self.offset				)]

     super().__init__(
         lowercase_  ,     tokenizer_file=lowercase_  ,     pad_token=lowercase_  ,     eos_token=lowercase_  ,     unk_token=lowercase_  ,     mask_token=lowercase_  ,     mask_token_sent=lowercase_  ,     offset=lowercase_  ,     additional_special_tokens=lowercase_  ,     **lowercase_  ,     )
     snake_case_ :     List[Any]				=	vocab_file
     snake_case_ :     List[Any]				=	False if not self.vocab_file else True

 def 						_snake_case					(     self		:       str  ,     lowercase_		:       Union[str, Any]				):
     snake_case_ :     Any				=	set(self.all_special_ids				)  # call it once instead of inside list comp
     all_special_ids.remove(self.unk_token_id				)  # <unk> is only sometimes special

     if all_special_ids != set(range(len(self.additional_special_tokens				) + 3				)				):
         raise ValueError(
             '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
             f" {len(self.additional_special_tokens				)} additional_special_tokens, but got {all_special_ids}"				)

     return [1 if x in all_special_ids else 0 for x in seq]

 def 						_snake_case					(     self		:       int  ,     lowercase_		:       List  ,     lowercase_		:       Optional[List] = None  ,     lowercase_		:       bool = False				):
     if already_has_special_tokens:
         return self._special_token_mask(lowercase_				)
     elif token_ids_a is None:
         return self._special_token_mask(lowercase_				) + [1]
     else:
         return self._special_token_mask(token_ids_a + token_ids_a				) + [1]

 def 						_snake_case					(     self		:       List[Any]  ,     lowercase_		:       Optional[int]  ,     lowercase_		:       str=None				):
     if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return token_ids_a + token_ids_a + [self.eos_token_id]






 def 						_snake_case					(     self		:       Optional[Any]  ,     lowercase_		:       str  ,     lowercase_		:       Optional[str] = None				):
     if not self.can_save_slow_tokenizer:
         raise ValueError(
             '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
             '''tokenizer.'''				)

     if not os.path.isdir(lowercase_				):
         logger.error(f"Vocabulary path ({save_directory}) should be a directory"				)
         return
     snake_case_ :     Dict				=	os.path.join(
         lowercase_  ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

     if os.path.abspath(self.vocab_file				) != os.path.abspath(lowercase_				):
         copyfile(self.vocab_file  ,     lowercase_				)

     return (out_vocab_file,)





