'''simple docstring'''






import gc
import random
import unittest

import numpy as np
import torch
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModel,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableUnCLIPImgaImgPipeline, UNetaDConditionModel
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from diffusers.pipelines.stable_diffusion.stable_unclip_image_normalizer import StableUnCLIPImageNormalizer
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import (
    enable_full_determinism,
    floats_tensor,
    load_image,
    load_numpy,
    require_torch_gpu,
    skip_mps,
    slow,
    torch_device,
)

from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
    assert_mean_pixel_difference,
)


enable_full_determinism()





class     A__							(       _UpperCamelCase					,		_UpperCamelCase					,		_UpperCamelCase					,		unittest.TestCase   ):





   """simple docstring"""





   UpperCamelCase_       :						Union[str, Any]    				=     StableUnCLIPImgaImgPipeline
   UpperCamelCase_       :						List[str]    				=     TEXT_GUIDED_IMAGE_VARIATION_PARAMS
   UpperCamelCase_       :						Union[str, Any]    				=     TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
   UpperCamelCase_       :						Union[str, Any]    				=     frozenset(
       []   )  # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess
   UpperCamelCase_       :						Optional[Any]    				=     frozenset([]   )




   def 			_lowerCAmelCase		( self     :		List[Any]			)       ->    Tuple:







      """simple docstring"""




      _UpperCAmelCase							:	str							=   3_2
      _UpperCAmelCase							:	Any							=   embedder_hidden_size

      # image encoding components

      _UpperCAmelCase							:	Dict							=   CLIPImageProcessor(crop_size=3_2						,     size=3_2			)

      torch.manual_seed(0			)
      _UpperCAmelCase							:	List[str]							=   CLIPVisionModelWithProjection(
          CLIPVisionConfig(
              hidden_size=lowerCAmelCase_						,     projection_dim=lowerCAmelCase_						,     num_hidden_layers=5						,     num_attention_heads=4						,     image_size=3_2						,     intermediate_size=3_7						,     patch_size=1						,     )			)

      # regular denoising components

      torch.manual_seed(0			)
      _UpperCAmelCase							:	Tuple							=   StableUnCLIPImageNormalizer(embedding_dim=lowerCAmelCase_			)
      _UpperCAmelCase							:	List[str]							=   DDPMScheduler(beta_schedule="squaredcos_cap_v2"			)

      torch.manual_seed(0			)
      _UpperCAmelCase							:	Any							=   CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"			)

      torch.manual_seed(0			)
      _UpperCAmelCase							:	Optional[Any]							=   CLIPTextModel(
          CLIPTextConfig(
              bos_token_id=0						,     eos_token_id=2						,     hidden_size=lowerCAmelCase_						,     projection_dim=3_2						,     intermediate_size=3_7						,     layer_norm_eps=1e-05						,     num_attention_heads=4						,     num_hidden_layers=5						,     pad_token_id=1						,     vocab_size=1_0_0_0						,     )			)

      torch.manual_seed(0			)
      _UpperCAmelCase							:	Optional[Any]							=   UNetaDConditionModel(
          sample_size=3_2						,     in_channels=4						,     out_channels=4						,     down_block_types=("CrossAttnDownBlock2D", "DownBlock2D")						,     up_block_types=("UpBlock2D", "CrossAttnUpBlock2D")						,     block_out_channels=(3_2, 6_4)						,     attention_head_dim=(2, 4)						,     class_embed_type="projection"						,     projection_class_embeddings_input_dim=embedder_projection_dim * 2						,     cross_attention_dim=lowerCAmelCase_						,     layers_per_block=1						,     upcast_attention=lowerCAmelCase_						,     use_linear_projection=lowerCAmelCase_						,     )

      torch.manual_seed(0			)
      _UpperCAmelCase							:	int							=   DDIMScheduler(
          beta_schedule="scaled_linear"						,     beta_start=0.0_0085						,     beta_end=0.012						,     prediction_type="v_prediction"						,     set_alpha_to_one=lowerCAmelCase_						,     steps_offset=1						,     )

      torch.manual_seed(0			)
      _UpperCAmelCase							:	int							=   AutoencoderKL()

      _UpperCAmelCase							:	List[Any]							=   {
          # image encoding components
          "feature_extractor": feature_extractor,
          "image_encoder": image_encoder.eval(),
          # image noising components
          "image_normalizer": image_normalizer.eval(),
          "image_noising_scheduler": image_noising_scheduler,
          # regular denoising components
          "tokenizer": tokenizer,
          "text_encoder": text_encoder.eval(),
          "unet": unet.eval(),
          "scheduler": scheduler,
          "vae": vae.eval(),
      }

      return components




   def 			_lowerCAmelCase		( self     :		Optional[Any]						,     lowerCAmelCase__     :		Dict						,     lowerCAmelCase__     :		Dict=0						,     lowerCAmelCase__     :		List[str]=True			)       ->    Optional[Any]:







      """simple docstring"""




      if str(lowerCAmelCase_			).startswith("mps"			):
         _UpperCAmelCase							:	str							=   torch.manual_seed(lowerCAmelCase_			)
      else:
         _UpperCAmelCase							:	Tuple							=   torch.Generator(device=lowerCAmelCase_			).manual_seed(lowerCAmelCase_			)

      _UpperCAmelCase							:	Optional[Any]							=   floats_tensor((1, 3, 3_2, 3_2)						,     rng=random.Random(lowerCAmelCase_			)			).to(lowerCAmelCase_			)

      if pil_image:
         _UpperCAmelCase							:	List[str]							=   input_image * 0.5 + 0.5
         _UpperCAmelCase							:	Any							=   input_image.clamp(0						,     1			)
         _UpperCAmelCase							:	Optional[int]							=   input_image.cpu().permute(0						,     2						,     3						,     1			).float().numpy()
         _UpperCAmelCase							:	int							=   DiffusionPipeline.numpy_to_pil(lowerCAmelCase_			)[0]

      return {
          "prompt": "An anime racoon running a marathon",
          "image": input_image,
          "generator": generator,
          "num_inference_steps": 2,
          "output_type": "np",
      }




   @skip_mps
   def 			_lowerCAmelCase		( self     :		Dict			)       ->    str:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   "cpu"  # ensure determinism for the device-dependent torch.Generator
      _UpperCAmelCase							:	Optional[Any]							=   self.get_dummy_components()
      _UpperCAmelCase							:	Optional[Any]							=   StableUnCLIPImgaImgPipeline(**lowerCAmelCase_			)
      _UpperCAmelCase							:	List[str]							=   sd_pipe.to(lowerCAmelCase_			)
      sd_pipe.set_progress_bar_config(disable=lowerCAmelCase_			)

      _UpperCAmelCase							:	Optional[Any]							=   self.get_dummy_inputs(lowerCAmelCase_			)
      inputs.update({"image_embeds": None}			)
      _UpperCAmelCase							:	Tuple							=   sd_pipe(**lowerCAmelCase_			).images
      _UpperCAmelCase							:	Tuple							=   image[0, -3:, -3:, -1]

      assert image.shape == (1, 3_2, 3_2, 3)
      _UpperCAmelCase							:	Optional[Any]							=   np.array([0.3872, 0.7224, 0.5601, 0.4741, 0.6872, 0.5814, 0.4636, 0.3867, 0.5078]			)

      assert np.abs(image_slice.flatten() - expected_slice			).max() < 1e-3




   def 			_lowerCAmelCase		( self     :		Optional[Any]			)       ->    Union[str, Any]:







      """simple docstring"""




      _UpperCAmelCase							:	List[str]							=   torch_device in ["cpu", "mps"]

      self._test_attention_slicing_forward_pass(test_max_difference=lowerCAmelCase_			)




   def 			_lowerCAmelCase		( self     :		int			)       ->    int:







      """simple docstring"""




      _UpperCAmelCase							:	List[Any]							=   torch_device in ["cpu", "mps"]

      self._test_inference_batch_single_identical(test_max_difference=lowerCAmelCase_			)




   @unittest.skipIf(
       torch_device != "cuda" or not is_xformers_available()						,     reason="XFormers attention is only available with CUDA and `xformers` installed"						,     )
   def 			_lowerCAmelCase		( self     :		List[Any]			)       ->    Union[str, Any]:







      """simple docstring"""




      self._test_xformers_attention_forwardGenerator_pass(test_max_difference=lowerCAmelCase_			)





@slow
@require_torch_gpu
class     A__							(       unittest.TestCase   ):





   """simple docstring"""





   def 			_lowerCAmelCase		( self     :		int			)       ->    Any:







      """simple docstring"""




      super().tearDown()
      gc.collect()
      torch.cuda.empty_cache()




   def 			_lowerCAmelCase		( self     :		Optional[Any]			)       ->    Any:







      """simple docstring"""




      _UpperCAmelCase							:	Dict							=   load_image(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/turtle.png"			)

      _UpperCAmelCase							:	Any							=   load_numpy(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_l_img2img_anime_turtle_fp16.npy"			)

      _UpperCAmelCase							:	Dict							=   StableUnCLIPImgaImgPipeline.from_pretrained(
          "fusing/stable-unclip-2-1-l-img2img"						,     torch_dtype=torch.floataa			)
      pipe.to(lowerCAmelCase_			)
      pipe.set_progress_bar_config(disable=lowerCAmelCase_			)
      # stable unclip will oom when integration tests are run on a V100,
      # so turn on memory savings
      pipe.enable_attention_slicing()
      pipe.enable_sequential_cpu_offload()

      _UpperCAmelCase							:	Optional[int]							=   torch.Generator(device="cpu"			).manual_seed(0			)
      _UpperCAmelCase							:	List[str]							=   pipe(lowerCAmelCase_						,     "anime turle"						,     generator=lowerCAmelCase_						,     output_type="np"			)

      _UpperCAmelCase							:	Optional[Any]							=   output.images[0]

      assert image.shape == (7_6_8, 7_6_8, 3)

      assert_mean_pixel_difference(lowerCAmelCase_						,     lowerCAmelCase_			)




   def 			_lowerCAmelCase		( self     :		Optional[Any]			)       ->    List[Any]:







      """simple docstring"""




      _UpperCAmelCase							:	Any							=   load_image(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/turtle.png"			)

      _UpperCAmelCase							:	List[str]							=   load_numpy(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_h_img2img_anime_turtle_fp16.npy"			)

      _UpperCAmelCase							:	str							=   StableUnCLIPImgaImgPipeline.from_pretrained(
          "fusing/stable-unclip-2-1-h-img2img"						,     torch_dtype=torch.floataa			)
      pipe.to(lowerCAmelCase_			)
      pipe.set_progress_bar_config(disable=lowerCAmelCase_			)
      # stable unclip will oom when integration tests are run on a V100,
      # so turn on memory savings
      pipe.enable_attention_slicing()
      pipe.enable_sequential_cpu_offload()

      _UpperCAmelCase							:	int							=   torch.Generator(device="cpu"			).manual_seed(0			)
      _UpperCAmelCase							:	Optional[int]							=   pipe(lowerCAmelCase_						,     "anime turle"						,     generator=lowerCAmelCase_						,     output_type="np"			)

      _UpperCAmelCase							:	Tuple							=   output.images[0]

      assert image.shape == (7_6_8, 7_6_8, 3)

      assert_mean_pixel_difference(lowerCAmelCase_						,     lowerCAmelCase_			)




   def 			_lowerCAmelCase		( self     :		Optional[int]			)       ->    Optional[Any]:







      """simple docstring"""




      _UpperCAmelCase							:	Any							=   load_image(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/turtle.png"			)

      torch.cuda.empty_cache()
      torch.cuda.reset_max_memory_allocated()
      torch.cuda.reset_peak_memory_stats()

      _UpperCAmelCase							:	Any							=   StableUnCLIPImgaImgPipeline.from_pretrained(
          "fusing/stable-unclip-2-1-h-img2img"						,     torch_dtype=torch.floataa			)
      _UpperCAmelCase							:	int							=   pipe.to(lowerCAmelCase_			)
      pipe.set_progress_bar_config(disable=lowerCAmelCase_			)
      pipe.enable_attention_slicing()
      pipe.enable_sequential_cpu_offload()

      _UpperCAmelCase							:	int							=   pipe(
          lowerCAmelCase_						,     "anime turtle"						,     num_inference_steps=2						,     output_type="np"						,     )

      _UpperCAmelCase							:	int							=   torch.cuda.max_memory_allocated()
      # make sure that less than 7 GB is allocated
      assert mem_bytes < 7 * 1_0**9






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False
  a_        = True






  def 				lowercase		(				self :		Optional[Any]     )							->						List[str]:
     super().setUp()

     __lowerCAmelCase								=	[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         'こんにちは',
         'こん',
         'にちは',
         'ばんは',
         '##こん',
         '##にちは',
         '##ばんは',
         '世界',
         '##世界',
         '、',
         '##、',
         '。',
         '##。',
     ]

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		Tuple     )							->						str:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こんにちは 、 世界 。 こんばんは 、 世界 。'
     return input_text, output_text






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		str     )							->						Dict:
     __lowerCAmelCase	,	__lowerCAmelCase								=	self.get_input_output_texts(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode(lowerCAmelCase_					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.decode(lowerCAmelCase_					,  clean_up_tokenization_spaces=lowerCAmelCase_     )
     return text, ids






  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Optional[Any]     )							->						Optional[Any]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )






  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='mecab'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	MecabTokenizer(mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		List[Any]     )							->						int:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic_lite'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(do_lower_case=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(
            do_lower_case=lowerCAmelCase_					,  normalize_text=lowerCAmelCase_					,  mecab_option='-d /usr/local/lib/mecab/dic/jumandic'     )
     except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(normalize_text=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,  )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='sudachi'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='A'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国', '人', '参政', '権']     )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='B'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人', '参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						Union[str, Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='C'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		Dict     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(do_lower_case=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(normalize_text=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	SudachiTokenizer(trim_whitespace=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='jumanpp'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_jumanpp
  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	JumanppTokenizer(do_lower_case=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Dict     )							->						Dict:
     __lowerCAmelCase								=	JumanppTokenizer(normalize_text=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	JumanppTokenizer(trim_whitespace=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Any:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'     )					,  ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						str:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	WordpieceTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こんにちは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは'     )					,  ['こん', '##ばんは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'     )					,  ['こん', '##ばんは', '[UNK]', 'こんにちは']     )






  def 				lowercase		(				self :		List[Any]     )							->						Tuple:
     __lowerCAmelCase								=	BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'     )
     __lowerCAmelCase								=	tokenizer.subword_tokenizer

     __lowerCAmelCase								=	subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']     )

     __lowerCAmelCase								=	subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False






  def 				lowercase		(				self :		Optional[Any]     )							->						Tuple:
     super().setUp()

     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		str					,  **lowerCAmelCase_ :		Tuple     )							->						Union[str, Any]:
     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,  subword_tokenizer_type='character'					,  **lowerCAmelCase_     )






  def 				lowercase		(				self :		Tuple					,  lowerCAmelCase_ :		Tuple     )							->						Optional[int]:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
     return input_text, output_text






  def 				lowercase		(				self :		Dict     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Any     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		List[Any]     )							->						int:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		str     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  subword_tokenizer_type='character'     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'     )
     self.assertListEqual(
         lowerCAmelCase_					,  ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']     )
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]     )






  def 				lowercase		(				self :		str     )							->						Optional[int]:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	CharacterTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こ', 'ん', 'に', 'ち', 'は']     )

     self.assertListEqual(tokenizer.tokenize('こんにちほ'     )					,  ['こ', 'ん', 'に', 'ち', '[UNK]']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						Union[str, Any]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     __lowerCAmelCase								=	AutoTokenizer.from_pretrained(lowerCAmelCase_     )
     self.assertIsInstance(lowerCAmelCase_					,  lowerCAmelCase_     )





class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )
     __lowerCAmelCase								=	'bert-base-cased'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertJapaneseTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )

