



"""simple docstring"""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_pegasus import PegasusTokenizer
else:
     lowercase__    :  int							    =	None


lowercase__    :  Any							    =	logging.get_logger(__name__)


lowercase__    :  List[str]							    =	'''▁'''

lowercase__    :  Optional[int]							    =	{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase__    :  str							    =	{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''},
    '''tokenizer_file''': {
        '''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'''
    },
}

lowercase__    :  List[Any]							    =	{
    '''google/pegasus-xsum''': 5_12,
}




class        _UpperCAmelCase       (			lowerCAmelCase__):
 _lowerCAmelCase   :			List[str]							=		VOCAB_FILES_NAMES
 _lowerCAmelCase   :			List[str]							=		PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase   :			Any							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase   :			Tuple							=		PegasusTokenizer
 _lowerCAmelCase   :			str							=		["""input_ids""", """attention_mask"""]

 def __init__(     self		:       Any  ,     lowercase_		:       Optional[Any]=None  ,     lowercase_		:       int=None  ,     lowercase_		:       Tuple="<pad>"  ,     lowercase_		:       int="</s>"  ,     lowercase_		:       Tuple="<unk>"  ,     lowercase_		:       str="<mask_2>"  ,     lowercase_		:       Optional[Any]="<mask_1>"  ,     lowercase_		:       str=None  ,     lowercase_		:       List[str]=103  ,     **lowercase_		:       List[Any]  ,     ):
     snake_case_ :     Dict				=	offset

     if additional_special_tokens is not None:
         if not isinstance(lowercase_  ,     lowercase_				):
             raise TypeError(
                 f"additional_special_tokens should be of type {type(lowercase_				)}, but is"
                 f" {type(lowercase_				)}"				)

         snake_case_ :     str				=	(
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f"<unk_{i}>" for i in range(len(lowercase_				)  ,     self.offset - 1				)
         ]

         if len(set(lowercase_				)				) != len(lowercase_				):
             raise ValueError(
                 '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
                 f" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."				)
         snake_case_ :     Union[str, Any]				=	additional_special_tokens_extended
     else:
         snake_case_ :     Dict				=	[mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f"<unk_{i}>" for i in range(2  ,     self.offset				)]

     super().__init__(
         lowercase_  ,     tokenizer_file=lowercase_  ,     pad_token=lowercase_  ,     eos_token=lowercase_  ,     unk_token=lowercase_  ,     mask_token=lowercase_  ,     mask_token_sent=lowercase_  ,     offset=lowercase_  ,     additional_special_tokens=lowercase_  ,     **lowercase_  ,     )
     snake_case_ :     List[Any]				=	vocab_file
     snake_case_ :     List[Any]				=	False if not self.vocab_file else True

 def 						_snake_case					(     self		:       str  ,     lowercase_		:       Union[str, Any]				):
     snake_case_ :     Any				=	set(self.all_special_ids				)  # call it once instead of inside list comp
     all_special_ids.remove(self.unk_token_id				)  # <unk> is only sometimes special

     if all_special_ids != set(range(len(self.additional_special_tokens				) + 3				)				):
         raise ValueError(
             '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
             f" {len(self.additional_special_tokens				)} additional_special_tokens, but got {all_special_ids}"				)

     return [1 if x in all_special_ids else 0 for x in seq]

 def 						_snake_case					(     self		:       int  ,     lowercase_		:       List  ,     lowercase_		:       Optional[List] = None  ,     lowercase_		:       bool = False				):
     if already_has_special_tokens:
         return self._special_token_mask(lowercase_				)
     elif token_ids_a is None:
         return self._special_token_mask(lowercase_				) + [1]
     else:
         return self._special_token_mask(token_ids_a + token_ids_a				) + [1]

 def 						_snake_case					(     self		:       List[Any]  ,     lowercase_		:       Optional[int]  ,     lowercase_		:       str=None				):
     if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return token_ids_a + token_ids_a + [self.eos_token_id]






 def 						_snake_case					(     self		:       Optional[Any]  ,     lowercase_		:       str  ,     lowercase_		:       Optional[str] = None				):
     if not self.can_save_slow_tokenizer:
         raise ValueError(
             '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
             '''tokenizer.'''				)

     if not os.path.isdir(lowercase_				):
         logger.error(f"Vocabulary path ({save_directory}) should be a directory"				)
         return
     snake_case_ :     Dict				=	os.path.join(
         lowercase_  ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

     if os.path.abspath(self.vocab_file				) != os.path.abspath(lowercase_				):
         copyfile(self.vocab_file  ,     lowercase_				)

     return (out_vocab_file,)










"""simple docstring"""

import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
     from transformers.trainer import DEFAULT_CALLBACKS

     from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel




class        _UpperCAmelCase       (			lowerCAmelCase__):

 def __init__(     self		:       Optional[int]				):
     snake_case_ :     str				=	[]

 def 						_snake_case					(     self		:       List[Any]  ,     lowercase_		:       Any  ,     lowercase_		:       Union[str, Any]  ,     lowercase_		:       List[str]  ,     **lowercase_		:       Tuple				):
     self.events.append('''on_init_end'''				)

 def 						_snake_case					(     self		:       List[Any]  ,     lowercase_		:       str  ,     lowercase_		:       Optional[int]  ,     lowercase_		:       List[str]  ,     **lowercase_		:       List[str]				):
     self.events.append('''on_train_begin'''				)

 def 						_snake_case					(     self		:       Any  ,     lowercase_		:       List[str]  ,     lowercase_		:       Tuple  ,     lowercase_		:       List[Any]  ,     **lowercase_		:       Optional[int]				):
     self.events.append('''on_train_end'''				)

 def 						_snake_case					(     self		:       str  ,     lowercase_		:       Optional[int]  ,     lowercase_		:       int  ,     lowercase_		:       Optional[Any]  ,     **lowercase_		:       List[Any]				):
     self.events.append('''on_epoch_begin'''				)

 def 						_snake_case					(     self		:       Tuple  ,     lowercase_		:       List[str]  ,     lowercase_		:       Dict  ,     lowercase_		:       Union[str, Any]  ,     **lowercase_		:       Optional[Any]				):
     self.events.append('''on_epoch_end'''				)

 def 						_snake_case					(     self		:       List[str]  ,     lowercase_		:       Optional[Any]  ,     lowercase_		:       Optional[Any]  ,     lowercase_		:       int  ,     **lowercase_		:       Optional[Any]				):
     self.events.append('''on_step_begin'''				)

 def 						_snake_case					(     self		:       int  ,     lowercase_		:       int  ,     lowercase_		:       Union[str, Any]  ,     lowercase_		:       List[Any]  ,     **lowercase_		:       List[str]				):
     self.events.append('''on_step_end'''				)

 def 						_snake_case					(     self		:       str  ,     lowercase_		:       int  ,     lowercase_		:       Dict  ,     lowercase_		:       List[str]  ,     **lowercase_		:       List[str]				):
     self.events.append('''on_evaluate'''				)

 def 						_snake_case					(     self		:       Dict  ,     lowercase_		:       Union[str, Any]  ,     lowercase_		:       Any  ,     lowercase_		:       List[Any]  ,     **lowercase_		:       str				):
     self.events.append('''on_predict'''				)

 def 						_snake_case					(     self		:       List[Any]  ,     lowercase_		:       Union[str, Any]  ,     lowercase_		:       List[Any]  ,     lowercase_		:       int  ,     **lowercase_		:       Union[str, Any]				):
     self.events.append('''on_save'''				)

 def 						_snake_case					(     self		:       str  ,     lowercase_		:       Tuple  ,     lowercase_		:       Optional[int]  ,     lowercase_		:       List[str]  ,     **lowercase_		:       Any				):
     self.events.append('''on_log'''				)






 def 						_snake_case					(     self		:       Dict  ,     lowercase_		:       Optional[int]  ,     lowercase_		:       List[str]  ,     lowercase_		:       Union[str, Any]  ,     **lowercase_		:       Optional[int]				):
     self.events.append('''on_prediction_step'''				)





@require_torch
class        _UpperCAmelCase       (			unittest.TestCase):

 def 						_snake_case					(     self		:       List[str]				):
     snake_case_ :     Tuple				=	tempfile.mkdtemp()

 def 						_snake_case					(     self		:       Tuple				):
     shutil.rmtree(self.output_dir				)

 def 						_snake_case					(     self		:       int  ,     lowercase_		:       Union[str, Any]=0  ,     lowercase_		:       Dict=0  ,     lowercase_		:       List[str]=64  ,     lowercase_		:       Union[str, Any]=64  ,     lowercase_		:       Union[str, Any]=None  ,     lowercase_		:       Any=False  ,     **lowercase_		:       List[Any]				):
     # disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
     # its set to False since the tests later on depend on its value.
     snake_case_ :     int				=	RegressionDataset(length=lowercase_				)
     snake_case_ :     Any				=	RegressionDataset(length=lowercase_				)
     snake_case_ :     int				=	RegressionModelConfig(a=lowercase_  ,     b=lowercase_				)
     snake_case_ :     Tuple				=	RegressionPreTrainedModel(lowercase_				)

     snake_case_ :     Any				=	TrainingArguments(self.output_dir  ,     disable_tqdm=lowercase_  ,     report_to=[]  ,     **lowercase_				)
     return Trainer(
         lowercase_  ,     lowercase_  ,     train_dataset=lowercase_  ,     eval_dataset=lowercase_  ,     callbacks=lowercase_  ,     )

 def 						_snake_case					(     self		:       Optional[int]  ,     lowercase_		:       Any  ,     lowercase_		:       List[Any]				):
     self.assertEqual(len(lowercase_				)  ,     len(lowercase_				)				)

     # Order doesn't matter
     snake_case_ :     Any				=	sorted(lowercase_  ,     key=lambda lowercase_				: cb.__name__ if isinstance(lowercase_  ,     lowercase_				) else cb.__class__.__name__				)
     snake_case_ :     List[str]				=	sorted(lowercase_  ,     key=lambda lowercase_				: cb.__name__ if isinstance(lowercase_  ,     lowercase_				) else cb.__class__.__name__				)

     for cba, cba in zip(lowercase_  ,     lowercase_				):
         if isinstance(lowercase_  ,     lowercase_				) and isinstance(lowercase_  ,     lowercase_				):
             self.assertEqual(lowercase_  ,     lowercase_				)
         elif isinstance(lowercase_  ,     lowercase_				) and not isinstance(lowercase_  ,     lowercase_				):
             self.assertEqual(lowercase_  ,     cba.__class__				)
         elif not isinstance(lowercase_  ,     lowercase_				) and isinstance(lowercase_  ,     lowercase_				):
             self.assertEqual(cba.__class__  ,     lowercase_				)
         else:
             self.assertEqual(lowercase_  ,     lowercase_				)

 def 						_snake_case					(     self		:       Optional[Any]  ,     lowercase_		:       Tuple				):
     snake_case_ :     Tuple				=	['''on_init_end''', '''on_train_begin''']
     snake_case_ :     List[Any]				=	0
     snake_case_ :     Union[str, Any]				=	len(trainer.get_eval_dataloader()				)
     snake_case_ :     List[Any]				=	['''on_prediction_step'''] * len(trainer.get_eval_dataloader()				) + ['''on_log''', '''on_evaluate''']
     for _ in range(trainer.state.num_train_epochs				):
         expected_events.append('''on_epoch_begin'''				)
         for _ in range(lowercase_				):
             step += 1
             expected_events += ["on_step_begin", "on_step_end"]
             if step % trainer.args.logging_steps == 0:
                 expected_events.append('''on_log'''				)
             if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                 expected_events += evaluation_events.copy()
             if step % trainer.args.save_steps == 0:
                 expected_events.append('''on_save'''				)
         expected_events.append('''on_epoch_end'''				)
         if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
             expected_events += evaluation_events.copy()
     expected_events += ["on_log", "on_train_end"]
     return expected_events

 def 						_snake_case					(     self		:       List[str]				):
     snake_case_ :     Union[str, Any]				=	self.get_trainer()
     snake_case_ :     Dict				=	DEFAULT_CALLBACKS.copy() + [ProgressCallback]
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

     # Callbacks passed at init are added to the default callbacks
     snake_case_ :     Optional[Any]				=	self.get_trainer(callbacks=[MyTestTrainerCallback]				)
     expected_callbacks.append(lowercase_				)
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

     # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
     snake_case_ :     Optional[int]				=	self.get_trainer(disable_tqdm=lowercase_				)
     snake_case_ :     List[Any]				=	DEFAULT_CALLBACKS.copy() + [PrinterCallback]
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

 def 						_snake_case					(     self		:       int				):
     snake_case_ :     int				=	DEFAULT_CALLBACKS.copy() + [ProgressCallback]
     snake_case_ :     List[Any]				=	self.get_trainer()

     # We can add, pop, or remove by class name
     trainer.remove_callback(lowercase_				)
     expected_callbacks.remove(lowercase_				)
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

     snake_case_ :     Dict				=	self.get_trainer()
     snake_case_ :     Optional[int]				=	trainer.pop_callback(lowercase_				)
     self.assertEqual(cb.__class__  ,     lowercase_				)
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

     trainer.add_callback(lowercase_				)
     expected_callbacks.insert(0  ,     lowercase_				)
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

     # We can also add, pop, or remove by instance
     snake_case_ :     Optional[int]				=	self.get_trainer()
     snake_case_ :     List[Any]				=	trainer.callback_handler.callbacks[0]
     trainer.remove_callback(lowercase_				)
     expected_callbacks.remove(lowercase_				)
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

     snake_case_ :     List[Any]				=	self.get_trainer()
     snake_case_ :     Optional[int]				=	trainer.callback_handler.callbacks[0]
     snake_case_ :     Optional[Any]				=	trainer.pop_callback(lowercase_				)
     self.assertEqual(lowercase_  ,     lowercase_				)
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)

     trainer.add_callback(lowercase_				)
     expected_callbacks.insert(0  ,     lowercase_				)
     self.check_callbacks_equality(trainer.callback_handler.callbacks  ,     lowercase_				)






 def 						_snake_case					(     self		:       List[Any]				):
     import warnings

     # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
     warnings.simplefilter(action='''ignore'''  ,     category=lowercase_				)

     snake_case_ :     int				=	self.get_trainer(callbacks=[MyTestTrainerCallback]				)
     trainer.train()
     snake_case_ :     Union[str, Any]				=	trainer.callback_handler.callbacks[-2].events
     self.assertEqual(lowercase_  ,     self.get_expected_events(lowercase_				)				)

     # Independent log/save/eval
     snake_case_ :     int				=	self.get_trainer(callbacks=[MyTestTrainerCallback]  ,     logging_steps=5				)
     trainer.train()
     snake_case_ :     str				=	trainer.callback_handler.callbacks[-2].events
     self.assertEqual(lowercase_  ,     self.get_expected_events(lowercase_				)				)

     snake_case_ :     List[Any]				=	self.get_trainer(callbacks=[MyTestTrainerCallback]  ,     save_steps=5				)
     trainer.train()
     snake_case_ :     int				=	trainer.callback_handler.callbacks[-2].events
     self.assertEqual(lowercase_  ,     self.get_expected_events(lowercase_				)				)

     snake_case_ :     List[Any]				=	self.get_trainer(callbacks=[MyTestTrainerCallback]  ,     eval_steps=5  ,     evaluation_strategy='''steps'''				)
     trainer.train()
     snake_case_ :     Union[str, Any]				=	trainer.callback_handler.callbacks[-2].events
     self.assertEqual(lowercase_  ,     self.get_expected_events(lowercase_				)				)

     snake_case_ :     Union[str, Any]				=	self.get_trainer(callbacks=[MyTestTrainerCallback]  ,     evaluation_strategy='''epoch'''				)
     trainer.train()
     snake_case_ :     Dict				=	trainer.callback_handler.callbacks[-2].events
     self.assertEqual(lowercase_  ,     self.get_expected_events(lowercase_				)				)

     # A bit of everything
     snake_case_ :     str				=	self.get_trainer(
         callbacks=[MyTestTrainerCallback]  ,     logging_steps=3  ,     save_steps=10  ,     eval_steps=5  ,     evaluation_strategy='''steps'''  ,     )
     trainer.train()
     snake_case_ :     str				=	trainer.callback_handler.callbacks[-2].events
     self.assertEqual(lowercase_  ,     self.get_expected_events(lowercase_				)				)

     # warning should be emitted for duplicated callbacks
     with patch('''transformers.trainer_callback.logger.warning'''				) as warn_mock:
         snake_case_ :     Dict				=	self.get_trainer(
             callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]  ,     )
         assert str(lowercase_				) in warn_mock.call_args[0][0]





