






'''simple docstring'''







import gc
import math
import unittest

import torch

from diffusers import UNetaDModel
from diffusers.utils import floats_tensor, logging, slow, torch_all_close, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


UpperCamelCase     :   Dict	      =		logging.get_logger(__name__)

enable_full_determinism()




class 		UpperCamelCase__     (a ,a ,unittest.TestCase      ):




		'''simple docstring'''





		_UpperCamelCase						=       UNetaDModel
		_UpperCamelCase						=       'sample'






		@property
		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	4
								lowerCamelCase__								=	3
								lowerCamelCase__								=	(32, 32)

								lowerCamelCase__								=	floats_tensor((batch_size, num_channels) + sizes							).to(_lowerCAmelCase							)
								lowerCamelCase__								=	torch.tensor([10]							).to(_lowerCAmelCase							)

								return {"sample": noise, "timestep": time_step}






		@property
		def       UpperCamelCase_       (   self							):
								return (3, 32, 32)






		@property
		def       UpperCamelCase_       (   self							):
								return (3, 32, 32)






		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	{
								    """block_out_channels""": (32, 64),
								    """down_block_types""": ("""DownBlock2D""", """AttnDownBlock2D"""),
								    """up_block_types""": ("""AttnUpBlock2D""", """UpBlock2D"""),
								    """attention_head_dim""": 3,
								    """out_channels""": 3,
								    """in_channels""": 3,
								    """layers_per_block""": 2,
								    """sample_size""": 32,
								}
								lowerCamelCase__								=	self.dummy_input
								return init_dict, inputs_dict




class 		UpperCamelCase__     (a ,a ,unittest.TestCase      ):




		'''simple docstring'''





		_UpperCamelCase						=       UNetaDModel
		_UpperCamelCase						=       'sample'






		@property
		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	4
								lowerCamelCase__								=	4
								lowerCamelCase__								=	(32, 32)

								lowerCamelCase__								=	floats_tensor((batch_size, num_channels) + sizes							).to(_lowerCAmelCase							)
								lowerCamelCase__								=	torch.tensor([10]							).to(_lowerCAmelCase							)

								return {"sample": noise, "timestep": time_step}






		@property
		def       UpperCamelCase_       (   self							):
								return (4, 32, 32)






		@property
		def       UpperCamelCase_       (   self							):
								return (4, 32, 32)






		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	{
								    """sample_size""": 32,
								    """in_channels""": 4,
								    """out_channels""": 4,
								    """layers_per_block""": 2,
								    """block_out_channels""": (32, 64),
								    """attention_head_dim""": 32,
								    """down_block_types""": ("""DownBlock2D""", """DownBlock2D"""),
								    """up_block_types""": ("""UpBlock2D""", """UpBlock2D"""),
								}
								lowerCamelCase__								=	self.dummy_input
								return init_dict, inputs_dict






		def       UpperCamelCase_       (   self							):
								lowerCamelCase__							,      lowerCamelCase__								=	UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update""" ,output_loading_info=_lowerCAmelCase							)

								self.assertIsNotNone(_lowerCAmelCase							)
								self.assertEqual(len(loading_info["""missing_keys"""]							) ,0							)

								model.to(_lowerCAmelCase							)
								lowerCamelCase__								=	model(**self.dummy_input							).sample

								assert image is not None, "Make sure output is not None"






		@unittest.skipIf(torch_device != """cuda""" ,"""This test is supposed to run on GPU"""							)
		def       UpperCamelCase_       (   self							):
								lowerCamelCase__							,      lowerCamelCase__								=	UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update""" ,output_loading_info=_lowerCAmelCase							)
								model.to(_lowerCAmelCase							)
								lowerCamelCase__								=	model(**self.dummy_input							).sample

								assert image is not None, "Make sure output is not None"






		@unittest.skipIf(torch_device != """cuda""" ,"""This test is supposed to run on GPU"""							)
		def       UpperCamelCase_       (   self							):
								# by defautl model loading will use accelerate as `low_cpu_mem_usage=True`
								lowerCamelCase__							,      lowerCamelCase__								=	UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update""" ,output_loading_info=_lowerCAmelCase							)
								model_accelerate.to(_lowerCAmelCase							)
								model_accelerate.eval()

								lowerCamelCase__								=	torch.randn(
								    1 ,model_accelerate.config.in_channels ,model_accelerate.config.sample_size ,model_accelerate.config.sample_size ,generator=torch.manual_seed(0							) ,)
								lowerCamelCase__								=	noise.to(_lowerCAmelCase							)
								lowerCamelCase__								=	torch.tensor([10] * noise.shape[0]							).to(_lowerCAmelCase							)

								lowerCamelCase__								=	model_accelerate(_lowerCAmelCase ,_lowerCAmelCase							)["""sample"""]

								# two models don't need to stay in the device at the same time
								del model_accelerate
								torch.cuda.empty_cache()
								gc.collect()

								lowerCamelCase__							,      lowerCamelCase__								=	UNetaDModel.from_pretrained(
								    """fusing/unet-ldm-dummy-update""" ,output_loading_info=_lowerCAmelCase ,low_cpu_mem_usage=_lowerCAmelCase							)
								model_normal_load.to(_lowerCAmelCase							)
								model_normal_load.eval()
								lowerCamelCase__								=	model_normal_load(_lowerCAmelCase ,_lowerCAmelCase							)["""sample"""]

								assert torch_all_close(_lowerCAmelCase ,_lowerCAmelCase ,rtol=1E-3							)






		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update"""							)
								model.eval()
								model.to(_lowerCAmelCase							)

								lowerCamelCase__								=	torch.randn(
								    1 ,model.config.in_channels ,model.config.sample_size ,model.config.sample_size ,generator=torch.manual_seed(0							) ,)
								lowerCamelCase__								=	noise.to(_lowerCAmelCase							)
								lowerCamelCase__								=	torch.tensor([10] * noise.shape[0]							).to(_lowerCAmelCase							)

								with torch.no_grad():
														lowerCamelCase__								=	model(_lowerCAmelCase ,_lowerCAmelCase							).sample

								lowerCamelCase__								=	output[0, -1, -3:, -3:].flatten().cpu()
								# fmt: off
								lowerCamelCase__								=	torch.tensor([-13.3258, -20.1100, -15.9873, -17.6617, -23.0596, -17.9419, -13.3675, -16.1889, -12.3800]							)
								# fmt: on

								self.assertTrue(torch_all_close(_lowerCAmelCase ,_lowerCAmelCase ,rtol=1E-3							)							)




class 		UpperCamelCase__     (a ,a ,unittest.TestCase      ):




		'''simple docstring'''





		_UpperCamelCase						=       UNetaDModel
		_UpperCamelCase						=       'sample'






		@property
		def       UpperCamelCase_       (   self ,_lowerCAmelCase=(32, 32)							):
								lowerCamelCase__								=	4
								lowerCamelCase__								=	3

								lowerCamelCase__								=	floats_tensor((batch_size, num_channels) + sizes							).to(_lowerCAmelCase							)
								lowerCamelCase__								=	torch.tensor(batch_size * [10]							).to(dtype=torch.intaa ,device=_lowerCAmelCase							)

								return {"sample": noise, "timestep": time_step}






		@property
		def       UpperCamelCase_       (   self							):
								return (3, 32, 32)






		@property
		def       UpperCamelCase_       (   self							):
								return (3, 32, 32)






		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	{
								    """block_out_channels""": [32, 64, 64, 64],
								    """in_channels""": 3,
								    """layers_per_block""": 1,
								    """out_channels""": 3,
								    """time_embedding_type""": """fourier""",
								    """norm_eps""": 1E-6,
								    """mid_block_scale_factor""": math.sqrt(2.0							),
								    """norm_num_groups""": None,
								    """down_block_types""": [
								        """SkipDownBlock2D""",
								        """AttnSkipDownBlock2D""",
								        """SkipDownBlock2D""",
								        """SkipDownBlock2D""",
								    ],
								    """up_block_types""": [
								        """SkipUpBlock2D""",
								        """SkipUpBlock2D""",
								        """AttnSkipUpBlock2D""",
								        """SkipUpBlock2D""",
								    ],
								}
								lowerCamelCase__								=	self.dummy_input
								return init_dict, inputs_dict






		@slow
		def       UpperCamelCase_       (   self							):
								lowerCamelCase__							,      lowerCamelCase__								=	UNetaDModel.from_pretrained("""google/ncsnpp-celebahq-256""" ,output_loading_info=_lowerCAmelCase							)
								self.assertIsNotNone(_lowerCAmelCase							)
								self.assertEqual(len(loading_info["""missing_keys"""]							) ,0							)

								model.to(_lowerCAmelCase							)
								lowerCamelCase__								=	self.dummy_input
								lowerCamelCase__								=	floats_tensor((4, 3) + (2_56, 2_56)							).to(_lowerCAmelCase							)
								lowerCamelCase__								=	noise
								lowerCamelCase__								=	model(**_lowerCAmelCase							)

								assert image is not None, "Make sure output is not None"






		@slow
		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	UNetaDModel.from_pretrained("""google/ncsnpp-celebahq-256"""							)
								model.to(_lowerCAmelCase							)

								lowerCamelCase__								=	4
								lowerCamelCase__								=	3
								lowerCamelCase__								=	(2_56, 2_56)

								lowerCamelCase__								=	torch.ones((batch_size, num_channels) + sizes							).to(_lowerCAmelCase							)
								lowerCamelCase__								=	torch.tensor(batch_size * [1E-4]							).to(_lowerCAmelCase							)

								with torch.no_grad():
														lowerCamelCase__								=	model(_lowerCAmelCase ,_lowerCAmelCase							).sample

								lowerCamelCase__								=	output[0, -3:, -3:, -1].flatten().cpu()
								# fmt: off
								lowerCamelCase__								=	torch.tensor([-4842.8691, -6499.6631, -3800.1953, -7978.2686, -1_0980.7129, -2_0028.8535, 8148.2822, 2342.2905, 567.7608]							)
								# fmt: on

								self.assertTrue(torch_all_close(_lowerCAmelCase ,_lowerCAmelCase ,rtol=1E-2							)							)






		def       UpperCamelCase_       (   self							):
								lowerCamelCase__								=	UNetaDModel.from_pretrained("""fusing/ncsnpp-ffhq-ve-dummy-update"""							)
								model.to(_lowerCAmelCase							)

								lowerCamelCase__								=	4
								lowerCamelCase__								=	3
								lowerCamelCase__								=	(32, 32)

								lowerCamelCase__								=	torch.ones((batch_size, num_channels) + sizes							).to(_lowerCAmelCase							)
								lowerCamelCase__								=	torch.tensor(batch_size * [1E-4]							).to(_lowerCAmelCase							)

								with torch.no_grad():
														lowerCamelCase__								=	model(_lowerCAmelCase ,_lowerCAmelCase							).sample

								lowerCamelCase__								=	output[0, -3:, -3:, -1].flatten().cpu()
								# fmt: off
								lowerCamelCase__								=	torch.tensor([-0.0325, -0.0900, -0.0869, -0.0332, -0.0725, -0.0270, -0.0101, 0.0227, 0.0256]							)
								# fmt: on

								self.assertTrue(torch_all_close(_lowerCAmelCase ,_lowerCAmelCase ,rtol=1E-2							)							)






		def       UpperCamelCase_       (   self							):
								# not required for this model
								pass









'''simple docstring'''
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 							_UpperCAmelCase    (		snake_case					,       unittest.TestCase      ):
    __lowerCamelCase: Optional[int]									=     BertTokenizer
    __lowerCamelCase: Optional[int]									=     BertTokenizerFast
    __lowerCamelCase: Union[str, Any]									=     True
    __lowerCamelCase: int									=     True
    __lowerCamelCase: Optional[Any]									=     filter_non_english




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       super().setUp()

       lowercase_						:     Optional[Any]							     =  [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       lowercase_						:     Optional[Any]							     =  os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )




    def 					lowerCAmelCase__					(      self	:							Any	,  a	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  "UNwant\u00E9d,running"
       lowercase_						:     List[str]							     =  "unwanted, running"
       return input_text, output_text




    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  self.tokenizer_class(self.vocab_file      )

       lowercase_						:     Union[str, Any]							     =  tokenizer.tokenize("UNwant\u00E9d,running"      )
       self.assertListEqual(a	,  ["un", "##want", "##ed", ",", "runn", "##ing"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(a      )	,  [9, 6, 7, 1_2, 1_0, 1_1]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       if not self.test_rust_tokenizer:
          return

       lowercase_						:     Optional[Any]							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       lowercase_						:     int							     =  "UNwant\u00E9d,running"

       lowercase_						:     Dict							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     List[Any]							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     List[Any]							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Optional[int]							     =  self.get_rust_tokenizer()
       lowercase_						:     Optional[int]							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )

       # With lower casing
       lowercase_						:     Tuple							     =  self.get_tokenizer(do_lower_case=a      )
       lowercase_						:     Dict							     =  self.get_rust_tokenizer(do_lower_case=a      )

       lowercase_						:     List[Any]							     =  "UNwant\u00E9d,running"

       lowercase_						:     str							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Any							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     str							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Union[str, Any]							     =  self.get_rust_tokenizer()
       lowercase_						:     Tuple							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      )	,  ["ah", "\u535A", "\u63A8", "zz"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["hello", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hällo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["h\u00E9llo"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     str							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     int							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HäLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Union[str, Any]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HaLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer(do_lower_case=a	,  never_split=["[UNK]"]      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer()
       lowercase_						:     Optional[int]							     =  "a\n'll !!to?'d of, can't."
       lowercase_						:     List[Any]							     =  ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
       self.assertListEqual(tokenizer.tokenize(a      )	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       lowercase_						:     int							     =  {}
       for i, token in enumerate(a      ):
          lowercase_						:     Any							     =  i
       lowercase_						:     Union[str, Any]							     =  WordpieceTokenizer(vocab=a	,  unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      )	,  []      )

       self.assertListEqual(tokenizer.tokenize("unwanted running"      )	,  ["un", "##want", "##ed", "runn", "##ing"]      )

       self.assertListEqual(tokenizer.tokenize("unwantedX running"      )	,  ["[UNK]", "runn", "##ing"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       self.assertTrue(_is_whitespace(" "      )      )
       self.assertTrue(_is_whitespace("\t"      )      )
       self.assertTrue(_is_whitespace("\r"      )      )
       self.assertTrue(_is_whitespace("\n"      )      )
       self.assertTrue(_is_whitespace("\u00A0"      )      )

       self.assertFalse(_is_whitespace("A"      )      )
       self.assertFalse(_is_whitespace("-"      )      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       self.assertTrue(_is_control("\u0005"      )      )

       self.assertFalse(_is_control("A"      )      )
       self.assertFalse(_is_control(" "      )      )
       self.assertFalse(_is_control("\t"      )      )
       self.assertFalse(_is_control("\r"      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       self.assertTrue(_is_punctuation("-"      )      )
       self.assertTrue(_is_punctuation("$"      )      )
       self.assertTrue(_is_punctuation("`"      )      )
       self.assertTrue(_is_punctuation("."      )      )

       self.assertFalse(_is_punctuation("A"      )      )
       self.assertFalse(_is_punctuation(" "      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       lowercase_						:     Dict							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )

       self.assertListEqual(
           [rust_tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )




    @slow
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     str							     =  self.tokenizer_class.from_pretrained("bert-base-uncased"      )

       lowercase_						:     Tuple							     =  tokenizer.encode("sequence builders"	,  add_special_tokens=a      )
       lowercase_						:     Union[str, Any]							     =  tokenizer.encode("multi-sequence build"	,  add_special_tokens=a      )

       lowercase_						:     Dict							     =  tokenizer.build_inputs_with_special_tokens(a      )
       lowercase_						:     Optional[Any]							     =  tokenizer.build_inputs_with_special_tokens(a	,  a      )

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Dict							     =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
             lowercase_						:     int							     =  tokenizer_r.encode_plus(
                 a	,  return_attention_mask=a	,  return_token_type_ids=a	,  return_offsets_mapping=a	,  add_special_tokens=a	,  )

             lowercase_						:     Tuple							     =  tokenizer_r.do_lower_case if hasattr(a	,  "do_lower_case"      ) else False
             lowercase_						:     List[Any]							     =  (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "Allen"),
                     ((2_1, 2_3), "##NL"),
                     ((2_3, 2_4), "##P"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "allen"),
                     ((2_1, 2_3), "##nl"),
                     ((2_3, 2_4), "##p"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]	,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
             self.assertEqual([e[0] for e in expected_results]	,  tokens["offset_mapping"]      )





    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  ["的", "人", "有"]
       lowercase_						:     int							     =  "".join(a      )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  True
             lowercase_						:     Dict							     =  self.tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )

             lowercase_						:     str							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )

             lowercase_						:     Tuple							     =  False
             lowercase_						:     Union[str, Any]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )

             lowercase_						:     int							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     List[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase_						:     int							     =  [
                 f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a      )
             ]
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )




