import tempfile

import torch

from diffusers import (
    DEISMultistepScheduler,
    DPMSolverMultistepScheduler,
    DPMSolverSinglestepScheduler,
    UniPCMultistepScheduler,
)

from .test_schedulers import SchedulerCommonTest

class 					_lowercase    (     UpperCAmelCase__			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Tuple          =    (DPMSolverSinglestepScheduler,)
    SCREAMING_SNAKE_CASE:				List[str]          =    (('num_inference_steps', 25),)




    def      _a    (    self       ,						**lowerCamelCase__ ):
       lowerCAmelCase_:							Tuple       =	{
           "num_train_timesteps": 1_000,
           "beta_start": 0.0_0_0_1,
           "beta_end": 0.0_2,
           "beta_schedule": "linear",
           "solver_order": 2,
           "prediction_type": "epsilon",
           "thresholding": False,
           "sample_max_value": 1.0,
           "algorithm_type": "dpmsolver++",
           "solver_type": "midpoint",
           "lambda_min_clipped": -float("inf" ),
           "variance_type": None,
       }

       config.update(**lowerCamelCase__ )
       return config




    def      _a    (    self       ,						lowerCamelCase__=0       ,						**lowerCamelCase__ ):
       lowerCAmelCase_:							Optional[int]       =	dict(self.forward_default_kwargs )
       lowerCAmelCase_:							Optional[int]       =	kwargs.pop("num_inference_steps"       ,						lowerCamelCase__ )
       lowerCAmelCase_:							Tuple       =	self.dummy_sample
       lowerCAmelCase_:							Any       =	0.1 * sample
       lowerCAmelCase_:							Union[str, Any]       =	[residual + 0.2, residual + 0.1_5, residual + 0.1_0]

       for scheduler_class in self.scheduler_classes:
          lowerCAmelCase_:							int       =	self.get_scheduler_config(**lowerCamelCase__ )
          lowerCAmelCase_:							str       =	scheduler_class(**lowerCamelCase__ )
          scheduler.set_timesteps(lowerCamelCase__ )
          # copy over dummy past residuals
          lowerCAmelCase_:							Tuple       =	dummy_past_residuals[: scheduler.config.solver_order]

          with tempfile.TemporaryDirectory() as tmpdirname:
             scheduler.save_config(lowerCamelCase__ )
             lowerCAmelCase_:							List[Any]       =	scheduler_class.from_pretrained(lowerCamelCase__ )
             new_scheduler.set_timesteps(lowerCamelCase__ )
             # copy over dummy past residuals
             lowerCAmelCase_:							str       =	dummy_past_residuals[: new_scheduler.config.solver_order]

          lowerCAmelCase_  ,	lowerCAmelCase_:							Union[str, Any]       =	sample, sample
          for t in range(lowerCamelCase__       ,						time_step + scheduler.config.solver_order + 1 ):
             lowerCAmelCase_:							Optional[int]       =	scheduler.step(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						**lowerCamelCase__ ).prev_sample
             lowerCAmelCase_:							List[Any]       =	new_scheduler.step(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						**lowerCamelCase__ ).prev_sample

             assert torch.sum(torch.abs(output - new_output ) ) < 1E-5, "Scheduler outputs are not identical"




    def      _a    (    self ):
       pass




    def      _a    (    self       ,						lowerCamelCase__=0       ,						**lowerCamelCase__ ):
       lowerCAmelCase_:							List[Any]       =	dict(self.forward_default_kwargs )
       lowerCAmelCase_:							Dict       =	kwargs.pop("num_inference_steps"       ,						lowerCamelCase__ )
       lowerCAmelCase_:							int       =	self.dummy_sample
       lowerCAmelCase_:							List[Any]       =	0.1 * sample
       lowerCAmelCase_:							int       =	[residual + 0.2, residual + 0.1_5, residual + 0.1_0]

       for scheduler_class in self.scheduler_classes:
          lowerCAmelCase_:							List[str]       =	self.get_scheduler_config()
          lowerCAmelCase_:							List[Any]       =	scheduler_class(**lowerCamelCase__ )
          scheduler.set_timesteps(lowerCamelCase__ )

          # copy over dummy past residuals (must be after setting timesteps)
          lowerCAmelCase_:							Any       =	dummy_past_residuals[: scheduler.config.solver_order]

          with tempfile.TemporaryDirectory() as tmpdirname:
             scheduler.save_config(lowerCamelCase__ )
             lowerCAmelCase_:							int       =	scheduler_class.from_pretrained(lowerCamelCase__ )
             # copy over dummy past residuals
             new_scheduler.set_timesteps(lowerCamelCase__ )

             # copy over dummy past residual (must be after setting timesteps)
             lowerCAmelCase_:							str       =	dummy_past_residuals[: new_scheduler.config.solver_order]

          lowerCAmelCase_:							List[str]       =	scheduler.step(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						**lowerCamelCase__ ).prev_sample
          lowerCAmelCase_:							Optional[int]       =	new_scheduler.step(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						**lowerCamelCase__ ).prev_sample

          assert torch.sum(torch.abs(output - new_output ) ) < 1E-5, "Scheduler outputs are not identical"




    def      _a    (    self       ,						lowerCamelCase__=None       ,						**lowerCamelCase__ ):
       if scheduler is None:
          lowerCAmelCase_:							str       =	self.scheduler_classes[0]
          lowerCAmelCase_:							Optional[int]       =	self.get_scheduler_config(**lowerCamelCase__ )
          lowerCAmelCase_:							Optional[int]       =	scheduler_class(**lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	self.scheduler_classes[0]
       lowerCAmelCase_:							List[str]       =	self.get_scheduler_config(**lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	scheduler_class(**lowerCamelCase__ )

       lowerCAmelCase_:							List[Any]       =	10
       lowerCAmelCase_:							Dict       =	self.dummy_model()
       lowerCAmelCase_:							List[Any]       =	self.dummy_sample_deter
       scheduler.set_timesteps(lowerCamelCase__ )

       for i, t in enumerate(scheduler.timesteps ):
          lowerCAmelCase_:							Tuple       =	model(lowerCamelCase__       ,						lowerCamelCase__ )
          lowerCAmelCase_:							Union[str, Any]       =	scheduler.step(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ).prev_sample

       return sample




    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	DPMSolverSinglestepScheduler(**self.get_scheduler_config() )
       lowerCAmelCase_:							Dict       =	50
       lowerCAmelCase_:							Optional[int]       =	self.dummy_model()
       lowerCAmelCase_:							List[str]       =	self.dummy_sample_deter
       scheduler.set_timesteps(lowerCamelCase__ )

       # make sure that the first t is uneven
       for i, t in enumerate(scheduler.timesteps[3:] ):
          lowerCAmelCase_:							Optional[Any]       =	model(lowerCamelCase__       ,						lowerCamelCase__ )
          lowerCAmelCase_:							Any       =	scheduler.step(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ).prev_sample

       lowerCAmelCase_:							Optional[Any]       =	torch.mean(torch.abs(lowerCamelCase__ ) )

       assert abs(result_mean.item() - 0.2_5_7_4 ) < 1E-3




    def      _a    (    self ):
       for timesteps in [25, 50, 100, 999, 1_000]:
          self.check_over_configs(num_train_timesteps=lowerCamelCase__ )




    def      _a    (    self ):
       # make sure that iterating over schedulers with same config names gives same results
       # for defaults
       lowerCAmelCase_:							List[Any]       =	DPMSolverSinglestepScheduler(**self.get_scheduler_config() )
       lowerCAmelCase_:							Optional[Any]       =	self.full_loop(scheduler=lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	torch.mean(torch.abs(lowerCamelCase__ ) )

       assert abs(result_mean.item() - 0.2_7_9_1 ) < 1E-3

       lowerCAmelCase_:							int       =	DEISMultistepScheduler.from_config(scheduler.config )
       lowerCAmelCase_:							Any       =	DPMSolverMultistepScheduler.from_config(scheduler.config )
       lowerCAmelCase_:							List[Any]       =	UniPCMultistepScheduler.from_config(scheduler.config )
       lowerCAmelCase_:							Optional[int]       =	DPMSolverSinglestepScheduler.from_config(scheduler.config )

       lowerCAmelCase_:							Any       =	self.full_loop(scheduler=lowerCamelCase__ )
       lowerCAmelCase_:							int       =	torch.mean(torch.abs(lowerCamelCase__ ) )

       assert abs(result_mean.item() - 0.2_7_9_1 ) < 1E-3




    def      _a    (    self ):
       self.check_over_configs(thresholding=lowerCamelCase__ )
       for order in [1, 2, 3]:
          for solver_type in ["midpoint", "heun"]:
             for threshold in [0.5, 1.0, 2.0]:
                for prediction_type in ["epsilon", "sample"]:
                   self.check_over_configs(
                       thresholding=lowerCamelCase__       ,						prediction_type=lowerCamelCase__       ,						sample_max_value=lowerCamelCase__       ,						algorithm_type="dpmsolver++"       ,						solver_order=lowerCamelCase__       ,						solver_type=lowerCamelCase__       ,						)




    def      _a    (    self ):
       for prediction_type in ["epsilon", "v_prediction"]:
          self.check_over_configs(prediction_type=lowerCamelCase__ )




    def      _a    (    self ):
       for algorithm_type in ["dpmsolver", "dpmsolver++"]:
          for solver_type in ["midpoint", "heun"]:
             for order in [1, 2, 3]:
                for prediction_type in ["epsilon", "sample"]:
                   self.check_over_configs(
                       solver_order=lowerCamelCase__       ,						solver_type=lowerCamelCase__       ,						prediction_type=lowerCamelCase__       ,						algorithm_type=lowerCamelCase__       ,						)
                   lowerCAmelCase_:							str       =	self.full_loop(
                       solver_order=lowerCamelCase__       ,						solver_type=lowerCamelCase__       ,						prediction_type=lowerCamelCase__       ,						algorithm_type=lowerCamelCase__       ,						)
                   assert not torch.isnan(lowerCamelCase__ ).any(), "Samples have nan numbers"




    def      _a    (    self ):
       self.check_over_configs(lower_order_final=lowerCamelCase__ )
       self.check_over_configs(lower_order_final=lowerCamelCase__ )




    def      _a    (    self ):
       self.check_over_configs(lambda_min_clipped=-float("inf" ) )
       self.check_over_configs(lambda_min_clipped=-5.1 )




    def      _a    (    self ):
       self.check_over_configs(variance_type=lowerCamelCase__ )
       self.check_over_configs(variance_type="learned_range" )




    def      _a    (    self ):
       for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1_000]:
          self.check_over_forward(num_inference_steps=lowerCamelCase__       ,						time_step=0 )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	self.full_loop()
       lowerCAmelCase_:							List[Any]       =	torch.mean(torch.abs(lowerCamelCase__ ) )

       assert abs(result_mean.item() - 0.2_7_9_1 ) < 1E-3




    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	self.full_loop(use_karras_sigmas=lowerCamelCase__ )
       lowerCAmelCase_:							Union[str, Any]       =	torch.mean(torch.abs(lowerCamelCase__ ) )

       assert abs(result_mean.item() - 0.2_2_4_8 ) < 1E-3




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	self.full_loop(prediction_type="v_prediction" )
       lowerCAmelCase_:							List[Any]       =	torch.mean(torch.abs(lowerCamelCase__ ) )

       assert abs(result_mean.item() - 0.1_4_5_3 ) < 1E-3




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	self.full_loop(prediction_type="v_prediction"       ,						use_karras_sigmas=lowerCamelCase__ )
       lowerCAmelCase_:							Optional[int]       =	torch.mean(torch.abs(lowerCamelCase__ ) )

       assert abs(result_mean.item() - 0.0_6_4_9 ) < 1E-3




    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	self.scheduler_classes[0]
       lowerCAmelCase_:							List[str]       =	self.get_scheduler_config(thresholding=lowerCamelCase__       ,						dynamic_thresholding_ratio=0 )
       lowerCAmelCase_:							Tuple       =	scheduler_class(**lowerCamelCase__ )

       lowerCAmelCase_:							Any       =	10
       lowerCAmelCase_:							Union[str, Any]       =	self.dummy_model()
       lowerCAmelCase_:							Optional[int]       =	self.dummy_sample_deter.half()
       scheduler.set_timesteps(lowerCamelCase__ )

       for i, t in enumerate(scheduler.timesteps ):
          lowerCAmelCase_:							Dict       =	model(lowerCamelCase__       ,						lowerCamelCase__ )
          lowerCAmelCase_:							List[Any]       =	scheduler.step(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ).prev_sample

       assert sample.dtype == torch.floataa
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Union[str, Any]          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Dict          =    False
    SCREAMING_SNAKE_CASE:				List[Any]          =    True




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "こんにちは",
           "こん",
           "にちは",
           "ばんは",
           "##こん",
           "##にちは",
           "##ばんは",
           "世界",
           "##世界",
           "、",
           "##、",
           "。",
           "##。",
       ]

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							Optional[int]       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	"こんにちは 、 世界 。 こんばんは 、 世界 。"
       return input_text, output_text




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							str       =	self.get_input_output_texts(lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode(lowerCamelCase__       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							List[str]       =	tokenizer.decode(lowerCamelCase__       ,						clean_up_tokenization_spaces=lowerCamelCase__ )
       return text, ids




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	self.tokenizer_class(self.vocab_file )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。" )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="mecab" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[Any]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Dict       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Union[str, Any]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	MecabTokenizer(mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							List[Any]       =	MecabTokenizer(mecab_dic="unidic_lite" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							str       =	MecabTokenizer(mecab_dic="unidic" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	MecabTokenizer(do_lower_case=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							Any       =	MecabTokenizer(
              do_lower_case=lowerCamelCase__       ,						normalize_text=lowerCamelCase__       ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic" )
       except RuntimeError:
          # if dict doesn't exist in the system, previous code raises this error.
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	MecabTokenizer(normalize_text=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="sudachi" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[str]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[int]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							List[str]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Optional[int]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="A" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国", "人", "参政", "権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="B" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人", "参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="C" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	SudachiTokenizer(do_lower_case=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(normalize_text=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	SudachiTokenizer(trim_whitespace=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							int       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="jumanpp" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							Any       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Any       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(do_lower_case=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	JumanppTokenizer(normalize_text=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(trim_whitespace=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。" )       ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

       lowerCAmelCase_:							Tuple       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							List[Any]       =	i
       lowerCAmelCase_:							List[str]       =	WordpieceTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こんにちは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは" )       ,						["こん", "##ばんは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは" )       ,						["こん", "##ばんは", "[UNK]", "こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp" )
       lowerCAmelCase_:							Optional[Any]       =	tokenizer.subword_tokenizer

       lowerCAmelCase_:							List[str]       =	subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。" )
       self.assertListEqual(lowerCamelCase__       ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"] )

       lowerCAmelCase_:							Optional[int]       =	subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは" )
       self.assertListEqual(lowerCamelCase__       ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese" )

       lowerCAmelCase_:							int       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Optional[int]       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Tuple          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Optional[Any]          =    False




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							int       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						**lowerCamelCase__ ):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,						subword_tokenizer_type="character"       ,						**lowerCamelCase__ )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							str       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Tuple       =	"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
       return input_text, output_text




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class(self.vocab_file       ,						subword_tokenizer_type="character" )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。" )
       self.assertListEqual(
           lowerCamelCase__       ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"] )
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							List[str]       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							Optional[Any]       =	i
       lowerCAmelCase_:							List[Any]       =	CharacterTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こ", "ん", "に", "ち", "は"] )

       self.assertListEqual(tokenizer.tokenize("こんにちほ" )       ,						["こ", "ん", "に", "ち", "[UNK]"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char" )

       lowerCAmelCase_:							Any       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							Dict       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							Any       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	"cl-tohoku/bert-base-japanese"
       lowerCAmelCase_:							Optional[int]       =	AutoTokenizer.from_pretrained(lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )





class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	"cl-tohoku/bert-base-japanese"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )
       lowerCAmelCase_:							List[str]       =	"bert-base-cased"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertJapaneseTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )