





import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
      from transformers.trainer import DEFAULT_CALLBACKS

      from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel







class     __snake_case       (							__lowerCamelCase			):





       '''simple docstring'''







       def __init__(  self :    List[Any]		):
         __snake_case:						Union[str, Any]   =						[]




       def 							UpperCAmelCase__ (  self :    List[Any]							,				A :    Optional[Any]							,				A :    str							,				A :    Tuple							,				**A :    Union[str, Any]		):
         self.events.append("""on_init_end"""		)




       def 							UpperCAmelCase__ (  self :    List[str]							,				A :    List[Any]							,				A :    Optional[int]							,				A :    Tuple							,				**A :    Optional[int]		):
         self.events.append("""on_train_begin"""		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Tuple							,				A :    Dict							,				A :    Tuple							,				**A :    int		):
         self.events.append("""on_train_end"""		)




       def 							UpperCAmelCase__ (  self :    List[str]							,				A :    int							,				A :    Optional[Any]							,				A :    int							,				**A :    List[Any]		):
         self.events.append("""on_epoch_begin"""		)




       def 							UpperCAmelCase__ (  self :    str							,				A :    Optional[Any]							,				A :    Any							,				A :    Union[str, Any]							,				**A :    Any		):
         self.events.append("""on_epoch_end"""		)




       def 							UpperCAmelCase__ (  self :    Dict							,				A :    str							,				A :    List[str]							,				A :    int							,				**A :    Union[str, Any]		):
         self.events.append("""on_step_begin"""		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]							,				A :    Optional[int]							,				A :    Optional[int]							,				A :    Optional[Any]							,				**A :    List[Any]		):
         self.events.append("""on_step_end"""		)




       def 							UpperCAmelCase__ (  self :    Any							,				A :    Union[str, Any]							,				A :    Dict							,				A :    Optional[Any]							,				**A :    Optional[int]		):
         self.events.append("""on_evaluate"""		)




       def 							UpperCAmelCase__ (  self :    str							,				A :    List[str]							,				A :    Dict							,				A :    Dict							,				**A :    Union[str, Any]		):
         self.events.append("""on_predict"""		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Any							,				A :    str							,				A :    Tuple							,				**A :    str		):
         self.events.append("""on_save"""		)




       def 							UpperCAmelCase__ (  self :    List[str]							,				A :    Optional[Any]							,				A :    Any							,				A :    str							,				**A :    Optional[int]		):
         self.events.append("""on_log"""		)






       def 							UpperCAmelCase__ (  self :    Dict							,				A :    Optional[int]							,				A :    Optional[int]							,				A :    Tuple							,				**A :    str		):
         self.events.append("""on_prediction_step"""		)







@require_torch
class     __snake_case       (							unittest.TestCase			):





       '''simple docstring'''







       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         __snake_case:						Optional[int]   =						tempfile.mkdtemp()




       def 							UpperCAmelCase__ (  self :    List[str]		):
         shutil.rmtree(self.output_dir		)




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				A :    Tuple=0							,				A :    Optional[Any]=0							,				A :    Union[str, Any]=64							,				A :    Any=64							,				A :    Optional[Any]=None							,				A :    List[str]=False							,				**A :    str		):
         # disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
         # its set to False since the tests later on depend on its value.
         __snake_case:						Optional[Any]   =						RegressionDataset(length=A		)
         __snake_case:						List[Any]   =						RegressionDataset(length=A		)
         __snake_case:						List[str]   =						RegressionModelConfig(a=A							,				b=A		)
         __snake_case:						Dict   =						RegressionPreTrainedModel(A		)

         __snake_case:						int   =						TrainingArguments(self.output_dir							,				disable_tqdm=A							,				report_to=[]							,				**A		)
         return Trainer(
             A							,				A							,				train_dataset=A							,				eval_dataset=A							,				callbacks=A							,				)




       def 							UpperCAmelCase__ (  self :    str							,				A :    Union[str, Any]							,				A :    Any		):
         self.assertEqual(len(A		)							,				len(A		)		)

         # Order doesn't matter
         __snake_case:						Union[str, Any]   =						sorted(A							,				key=lambda A		: cb.__name__ if isinstance(A							,				A		) else cb.__class__.__name__		)
         __snake_case:						int   =						sorted(A							,				key=lambda A		: cb.__name__ if isinstance(A							,				A		) else cb.__class__.__name__		)

         for cba, cba in zip(A							,				A		):
           if isinstance(A							,				A		) and isinstance(A							,				A		):
             self.assertEqual(A							,				A		)
           elif isinstance(A							,				A		) and not isinstance(A							,				A		):
             self.assertEqual(A							,				cba.__class__		)
           elif not isinstance(A							,				A		) and isinstance(A							,				A		):
             self.assertEqual(cba.__class__							,				A		)
           else:
             self.assertEqual(A							,				A		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Union[str, Any]		):
         __snake_case:						int   =						["""on_init_end""", """on_train_begin"""]
         __snake_case:						List[Any]   =						0
         __snake_case:						List[Any]   =						len(trainer.get_eval_dataloader()		)
         __snake_case:						Union[str, Any]   =						["""on_prediction_step"""] * len(trainer.get_eval_dataloader()		) + ["""on_log""", """on_evaluate"""]
         for _ in range(trainer.state.num_train_epochs		):
           expected_events.append("""on_epoch_begin"""		)
           for _ in range(A		):
             step += 1
             expected_events += ["on_step_begin", "on_step_end"]
             if step % trainer.args.logging_steps == 0:
               expected_events.append("""on_log"""		)
             if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
               expected_events += evaluation_events.copy()
             if step % trainer.args.save_steps == 0:
               expected_events.append("""on_save"""		)
           expected_events.append("""on_epoch_end"""		)
           if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
             expected_events += evaluation_events.copy()
         expected_events += ["on_log", "on_train_end"]
         return expected_events




       def 							UpperCAmelCase__ (  self :    Optional[Any]		):
         __snake_case:						Union[str, Any]   =						self.get_trainer()
         __snake_case:						Tuple   =						DEFAULT_CALLBACKS.copy() + [ProgressCallback]
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)

         # Callbacks passed at init are added to the default callbacks
         __snake_case:						List[str]   =						self.get_trainer(callbacks=[MyTestTrainerCallback]		)
         expected_callbacks.append(A		)
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)

         # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
         __snake_case:						Any   =						self.get_trainer(disable_tqdm=A		)
         __snake_case:						str   =						DEFAULT_CALLBACKS.copy() + [PrinterCallback]
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)




       def 							UpperCAmelCase__ (  self :    List[Any]		):
         __snake_case:						Any   =						DEFAULT_CALLBACKS.copy() + [ProgressCallback]
         __snake_case:						Dict   =						self.get_trainer()

         # We can add, pop, or remove by class name
         trainer.remove_callback(A		)
         expected_callbacks.remove(A		)
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)

         __snake_case:						Optional[int]   =						self.get_trainer()
         __snake_case:						Any   =						trainer.pop_callback(A		)
         self.assertEqual(cb.__class__							,				A		)
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)

         trainer.add_callback(A		)
         expected_callbacks.insert(0							,				A		)
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)

         # We can also add, pop, or remove by instance
         __snake_case:						Optional[Any]   =						self.get_trainer()
         __snake_case:						List[str]   =						trainer.callback_handler.callbacks[0]
         trainer.remove_callback(A		)
         expected_callbacks.remove(A		)
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)

         __snake_case:						Optional[Any]   =						self.get_trainer()
         __snake_case:						Dict   =						trainer.callback_handler.callbacks[0]
         __snake_case:						str   =						trainer.pop_callback(A		)
         self.assertEqual(A							,				A		)
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)

         trainer.add_callback(A		)
         expected_callbacks.insert(0							,				A		)
         self.check_callbacks_equality(trainer.callback_handler.callbacks							,				A		)






       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         import warnings

         # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
         warnings.simplefilter(action="""ignore"""							,				category=A		)

         __snake_case:						Optional[int]   =						self.get_trainer(callbacks=[MyTestTrainerCallback]		)
         trainer.train()
         __snake_case:						List[Any]   =						trainer.callback_handler.callbacks[-2].events
         self.assertEqual(A							,				self.get_expected_events(A		)		)

         # Independent log/save/eval
         __snake_case:						Optional[int]   =						self.get_trainer(callbacks=[MyTestTrainerCallback]							,				logging_steps=5		)
         trainer.train()
         __snake_case:						int   =						trainer.callback_handler.callbacks[-2].events
         self.assertEqual(A							,				self.get_expected_events(A		)		)

         __snake_case:						str   =						self.get_trainer(callbacks=[MyTestTrainerCallback]							,				save_steps=5		)
         trainer.train()
         __snake_case:						List[str]   =						trainer.callback_handler.callbacks[-2].events
         self.assertEqual(A							,				self.get_expected_events(A		)		)

         __snake_case:						Any   =						self.get_trainer(callbacks=[MyTestTrainerCallback]							,				eval_steps=5							,				evaluation_strategy="""steps"""		)
         trainer.train()
         __snake_case:						List[str]   =						trainer.callback_handler.callbacks[-2].events
         self.assertEqual(A							,				self.get_expected_events(A		)		)

         __snake_case:						Tuple   =						self.get_trainer(callbacks=[MyTestTrainerCallback]							,				evaluation_strategy="""epoch"""		)
         trainer.train()
         __snake_case:						List[Any]   =						trainer.callback_handler.callbacks[-2].events
         self.assertEqual(A							,				self.get_expected_events(A		)		)

         # A bit of everything
         __snake_case:						List[Any]   =						self.get_trainer(
             callbacks=[MyTestTrainerCallback]							,				logging_steps=3							,				save_steps=10							,				eval_steps=5							,				evaluation_strategy="""steps"""							,				)
         trainer.train()
         __snake_case:						Dict   =						trainer.callback_handler.callbacks[-2].events
         self.assertEqual(A							,				self.get_expected_events(A		)		)

         # warning should be emitted for duplicated callbacks
         with patch("""transformers.trainer_callback.logger.warning"""		) as warn_mock:
           __snake_case:						List[Any]   =						self.get_trainer(
               callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]							,				)
           assert str(A		) in warn_mock.call_args[0][0]













import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __snake_case       (							__lowerCamelCase   ,			unittest.TestCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     GPTSanJapaneseTokenizer
       lowerCAmelCase__      		=     False
       lowerCAmelCase__      		=     {"""do_clean_text""": False, """add_prefix_space""": False}




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         super().setUp()

         # fmt: off
         __snake_case:						str   =						["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
         # fmt: on
         __snake_case:						List[Any]   =						{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
         __snake_case:						Optional[int]   =						{"""unk_token""": """<unk>"""}

         __snake_case:						Any   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
         __snake_case:						int   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""emoji_file"""]		)
         with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
         with open(self.emoji_file							,				"""w"""		) as emoji_writer:
           emoji_writer.write(json.dumps(A		)		)




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				**A :    int		):
         kwargs.update(self.special_tokens_map		)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,				**A		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Dict		):
         __snake_case:						Tuple   =						"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
         __snake_case:						str   =						"""こんにちは、世界。 \nこんばんは、世界。😀"""
         return input_text, output_text




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				A :    Optional[int]		):
         __snake_case  ,   __snake_case:						Optional[int]   =						self.get_input_output_texts(A		)
         __snake_case:						Optional[Any]   =						tokenizer.encode(A							,				add_special_tokens=A		)
         __snake_case:						int   =						tokenizer.decode(A							,				clean_up_tokenization_spaces=A		)
         return text, ids




       def 							UpperCAmelCase__ (  self :    int		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Dict		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         __snake_case:						Dict   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						List[str]   =						"""こんにちは、世界。　こんばんは、㔺界。"""
         __snake_case:						Optional[int]   =						["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
         __snake_case:						Any   =						tokenizer.tokenize(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids without special tokens
         __snake_case:						str   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids with special tokens
         __snake_case:						Optional[int]   =						tokens + [tokenizer.unk_token]
         __snake_case:						List[str]   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)




       def 							UpperCAmelCase__ (  self :    str		):
         __snake_case:						Union[str, Any]   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						Optional[Any]   =						"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
         __snake_case:						Optional[Any]   =						"""こんにちは、、、、世界。こんばんは、、、、世界。"""
         __snake_case:						Union[str, Any]   =						tokenizer.encode(A		)
         __snake_case:						List[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Any		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Any   =						"""こんにちは、世界。"""
         __snake_case:						Tuple   =						"""こんばんは、㔺界。😀"""
         __snake_case:						Optional[Any]   =						"""こんにちは、世界。こんばんは、世界。😀"""
         __snake_case:						int   =						tokenizer.encode(prefix_text + input_text		)
         __snake_case:						Union[str, Any]   =						tokenizer.encode(""""""							,				prefix_text=prefix_text + input_text		)
         __snake_case:						Tuple   =						tokenizer.encode(A							,				prefix_text=A		)
         __snake_case:						Union[str, Any]   =						tokenizer.decode(A		)
         __snake_case:						Dict   =						tokenizer.decode(A		)
         __snake_case:						Optional[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    List[str]		):
         __snake_case:						int   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Optional[int]   =						"""こんにちは、世界。"""
         __snake_case:						Any   =						"""こんばんは、㔺界。😀"""

         __snake_case:						Optional[int]   =						len(tokenizer.encode(A		)		) - 2
         __snake_case:						str   =						len(tokenizer.encode(A		)		) - 2

         __snake_case:						Dict   =						[1] + [0] * (len_prefix + len_text + 1)
         __snake_case:						str   =						[1] * (len_prefix + len_text + 1) + [0]
         __snake_case:						List[Any]   =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         __snake_case:						int   =						tokenizer(prefix_text + input_text		).token_type_ids
         __snake_case:						Optional[int]   =						tokenizer(""""""							,				prefix_text=prefix_text + input_text		).token_type_ids
         __snake_case:						Tuple   =						tokenizer(A							,				prefix_text=A		).token_type_ids
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Dict		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						int   =						tokenizer.encode("""あンいワ"""		)
         __snake_case:						Optional[int]   =						tokenizer.encode(""""""							,				prefix_text="""あンいワ"""		)
         __snake_case:						List[Any]   =						tokenizer.encode("""いワ"""							,				prefix_text="""あン"""		)

         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertNotEqual(A							,				A		)
         self.assertNotEqual(A							,				A		)
         self.assertEqual(x_token_a[1]							,				x_token_a[-1]		)  # SEG token
         self.assertEqual(x_token_a[1]							,				x_token_a[3]		)  # SEG token




       @slow
       def 							UpperCAmelCase__ (  self :    int		):
         __snake_case:						List[str]   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						Union[str, Any]   =						[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
         __snake_case:						Optional[int]   =						tokenizer(A							,				padding=A		)
         __snake_case:						int   =						tokenizer.batch_encode_plus(A							,				padding=A		)

         # fmt: off
         __snake_case:						List[str]   =						[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
         __snake_case:						int   =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         __snake_case:						Dict   =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids							,				A		)
         self.assertListEqual(x_token.token_type_ids							,				A		)
         self.assertListEqual(x_token.attention_mask							,				A		)
         self.assertListEqual(x_token_a.input_ids							,				A		)
         self.assertListEqual(x_token_a.token_type_ids							,				A		)
         self.assertListEqual(x_token_a.attention_mask							,				A		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass






       def 							UpperCAmelCase__ (  self :    List[str]		):
         # tokenizer has no padding token
         pass






