






'''simple docstring'''



import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

SCREAMING_SNAKE_CASE							=						[
    'cross_validation.py',
    'gradient_accumulation.py',
    'local_sgd.py',
    'multi_process_metrics.py',
    'memory.py',
    'automatic_gradient_accumulation.py',
    'fsdp_with_peak_mem_tracking.py',
    'deepspeed_with_config_support.py',
    'megatron_lm_gpt_pretraining.py',
]
class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		bool  ,   UpperCAmelCase      :		str = None  ,   UpperCAmelCase      :		list = None		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Optional[Any]						=None
       lowercase       : Any						=os.path.abspath(os.path.join('''examples'''  ,   '''by_feature'''		)		)
       lowercase       : List[str]						=os.path.abspath('''examples'''		)
       for item in os.listdir(UpperCAmelCase		):
        if item not in EXCLUDE_EXAMPLES:
         lowercase       : Union[str, Any]						=os.path.join(UpperCAmelCase  ,   UpperCAmelCase		)
         if os.path.isfile(UpperCAmelCase		) and ".py" in item_path:
          with self.subTest(
              tested_script=UpperCAmelCase  ,   feature_script=UpperCAmelCase  ,   tested_section='''main()''' if parser_only else '''training_function()'''  ,   ):
           lowercase       : List[Any]						=compare_against_test(
               os.path.join(UpperCAmelCase  ,   UpperCAmelCase		)  ,   UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)
           lowercase       : Optional[int]						='''\n'''.join(UpperCAmelCase		)
           if special_strings is not None:
            for string in special_strings:
             lowercase       : Optional[Any]						=diff.replace(UpperCAmelCase  ,   ''''''		)
           self.assertEqual(UpperCAmelCase  ,   ''''''		)






      def 					A__   (			self      :		Optional[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       self.one_complete_example('''complete_nlp_example.py'''  ,   UpperCAmelCase		)
       self.one_complete_example('''complete_nlp_example.py'''  ,   UpperCAmelCase		)






      def 					A__   (			self      :		str		)	->	Any:


       '''simple docstring'''

       lowercase       : str						=os.path.abspath(os.path.join('''examples'''  ,   '''cv_example.py'''		)		)
       lowercase       : str						=[
           ''' ''' * 16 + '''{\n\n''',
           ''' ''' * 20 + '''"accuracy": eval_metric["accuracy"],\n\n''',
           ''' ''' * 20 + '''"f1": eval_metric["f1"],\n\n''',
           ''' ''' * 20 + '''"train_loss": total_loss.item() / len(train_dataloader),\n\n''',
           ''' ''' * 20 + '''"epoch": epoch,\n\n''',
           ''' ''' * 16 + '''},\n\n''',
           ''' ''' * 16 + '''step=epoch,\n''',
           ''' ''' * 12,
           ''' ''' * 8 + '''for step, batch in enumerate(active_dataloader):\n''',
       ]
       self.one_complete_example('''complete_cv_example.py'''  ,   UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)
       self.one_complete_example('''complete_cv_example.py'''  ,   UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)





@mock.patch.dict(os.environ					,							{'''TESTING_MOCKED_DATALOADERS''': '''1'''}					)
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      UpperCamelCase_             =      False






      @classmethod
      def 					A__   (			cls      :		Optional[Any]		)	->	List[Any]:


       '''simple docstring'''

       super().setUpClass()
       lowercase       : List[str]						=tempfile.mkdtemp()
       lowercase       : Union[str, Any]						=os.path.join(cls._tmpdir  ,   '''default_config.yml'''		)

       write_basic_config(save_location=cls.configPath		)
       lowercase       : Any						=['''accelerate''', '''launch''', '''--config_file''', cls.configPath]






      @classmethod
      def 					A__   (			cls      :		Optional[Any]		)	->	Tuple:


       '''simple docstring'''

       super().tearDownClass()
       shutil.rmtree(cls._tmpdir		)






      def 					A__   (			self      :		List[str]		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Dict						=f'\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps epoch\n        --output_dir {self.tmpdir}\n        '.split()
       run_command(self._launch_args + testargs		)
       self.assertTrue(os.path.exists(os.path.join(self.tmpdir  ,   '''epoch_0'''		)		)		)






      def 					A__   (			self      :		int		)	->	Union[str, Any]:


       '''simple docstring'''

       lowercase       : Dict						=f'\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps 1\n        --output_dir {self.tmpdir}\n        '.split()
       lowercase       : str						=run_command(self._launch_args + testargs		)
       self.assertTrue(os.path.exists(os.path.join(self.tmpdir  ,   '''step_2'''		)		)		)






      def 					A__   (			self      :		Union[str, Any]		)	->	Dict:


       '''simple docstring'''

       lowercase       : Optional[int]						=f'\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir  ,   "epoch_0"		)}\n        '.split()
       lowercase       : Any						=run_command(self._launch_args + testargs  ,   return_stdout=UpperCAmelCase		)
       self.assertNotIn('''epoch 0:'''  ,   UpperCAmelCase		)
       self.assertIn('''epoch 1:'''  ,   UpperCAmelCase		)






      def 					A__   (			self      :		str		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : List[str]						=f'\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir  ,   "step_2"		)}\n        '.split()
       lowercase       : Tuple						=run_command(self._launch_args + testargs  ,   return_stdout=UpperCAmelCase		)
       if torch.cuda.is_available():
        lowercase       : List[Any]						=torch.cuda.device_count()
       else:
        lowercase       : Dict						=1
       if num_processes > 1:
        self.assertNotIn('''epoch 0:'''  ,   UpperCAmelCase		)
        self.assertIn('''epoch 1:'''  ,   UpperCAmelCase		)
       else:
        self.assertIn('''epoch 0:'''  ,   UpperCAmelCase		)
        self.assertIn('''epoch 1:'''  ,   UpperCAmelCase		)






      @slow
      def 					A__   (			self      :		Any		)	->	Union[str, Any]:


       '''simple docstring'''

       lowercase       : List[Any]						='''
        examples/by_feature/cross_validation.py
        --num_folds 2
        '''.split()
       with mock.patch.dict(os.environ  ,   {'''TESTING_MOCKED_DATALOADERS''': '''0'''}		):
        lowercase       : List[str]						=run_command(self._launch_args + testargs  ,   return_stdout=UpperCAmelCase		)
        lowercase       : List[Any]						=re.findall('''({.+})'''  ,   UpperCAmelCase		)
        lowercase       : Tuple						=[r for r in results if '''accuracy''' in r][-1]
        lowercase       : Union[str, Any]						=ast.literal_eval(UpperCAmelCase		)
        self.assertGreaterEqual(results['''accuracy''']  ,   0.7_5		)






      def 					A__   (			self      :		Any		)	->	Union[str, Any]:


       '''simple docstring'''

       lowercase       : Any						=['''examples/by_feature/multi_process_metrics.py''']
       run_command(self._launch_args + testargs		)






      @require_trackers
      @mock.patch.dict(os.environ  ,   {'''WANDB_MODE''': '''offline'''}		)
      def 					A__   (			self      :		int		)	->	List[str]:


       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmpdir:
        lowercase       : Tuple						=f'\n            examples/by_feature/tracking.py\n            --with_tracking\n            --project_dir {tmpdir}\n            '.split()
        run_command(self._launch_args + testargs		)
        self.assertTrue(os.path.exists(os.path.join(UpperCAmelCase  ,   '''tracking'''		)		)		)






      def 					A__   (			self      :		List[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Optional[Any]						=['''examples/by_feature/gradient_accumulation.py''']
       run_command(self._launch_args + testargs		)






      def 					A__   (			self      :		Dict		)	->	Tuple:


       '''simple docstring'''

       lowercase       : List[str]						=['''examples/by_feature/local_sgd.py''']
       run_command(self._launch_args + testargs		)











'''simple docstring'''
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 							_UpperCAmelCase    (		snake_case					,       unittest.TestCase      ):
    __lowerCamelCase: Optional[int]									=     BertTokenizer
    __lowerCamelCase: Optional[int]									=     BertTokenizerFast
    __lowerCamelCase: Union[str, Any]									=     True
    __lowerCamelCase: int									=     True
    __lowerCamelCase: Optional[Any]									=     filter_non_english




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       super().setUp()

       lowercase_						:     Optional[Any]							     =  [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       lowercase_						:     Optional[Any]							     =  os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )




    def 					lowerCAmelCase__					(      self	:							Any	,  a	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  "UNwant\u00E9d,running"
       lowercase_						:     List[str]							     =  "unwanted, running"
       return input_text, output_text




    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  self.tokenizer_class(self.vocab_file      )

       lowercase_						:     Union[str, Any]							     =  tokenizer.tokenize("UNwant\u00E9d,running"      )
       self.assertListEqual(a	,  ["un", "##want", "##ed", ",", "runn", "##ing"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(a      )	,  [9, 6, 7, 1_2, 1_0, 1_1]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       if not self.test_rust_tokenizer:
          return

       lowercase_						:     Optional[Any]							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       lowercase_						:     int							     =  "UNwant\u00E9d,running"

       lowercase_						:     Dict							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     List[Any]							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     List[Any]							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Optional[int]							     =  self.get_rust_tokenizer()
       lowercase_						:     Optional[int]							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )

       # With lower casing
       lowercase_						:     Tuple							     =  self.get_tokenizer(do_lower_case=a      )
       lowercase_						:     Dict							     =  self.get_rust_tokenizer(do_lower_case=a      )

       lowercase_						:     List[Any]							     =  "UNwant\u00E9d,running"

       lowercase_						:     str							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Any							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     str							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Union[str, Any]							     =  self.get_rust_tokenizer()
       lowercase_						:     Tuple							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      )	,  ["ah", "\u535A", "\u63A8", "zz"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["hello", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hällo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["h\u00E9llo"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     str							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     int							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HäLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Union[str, Any]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HaLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer(do_lower_case=a	,  never_split=["[UNK]"]      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer()
       lowercase_						:     Optional[int]							     =  "a\n'll !!to?'d of, can't."
       lowercase_						:     List[Any]							     =  ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
       self.assertListEqual(tokenizer.tokenize(a      )	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       lowercase_						:     int							     =  {}
       for i, token in enumerate(a      ):
          lowercase_						:     Any							     =  i
       lowercase_						:     Union[str, Any]							     =  WordpieceTokenizer(vocab=a	,  unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      )	,  []      )

       self.assertListEqual(tokenizer.tokenize("unwanted running"      )	,  ["un", "##want", "##ed", "runn", "##ing"]      )

       self.assertListEqual(tokenizer.tokenize("unwantedX running"      )	,  ["[UNK]", "runn", "##ing"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       self.assertTrue(_is_whitespace(" "      )      )
       self.assertTrue(_is_whitespace("\t"      )      )
       self.assertTrue(_is_whitespace("\r"      )      )
       self.assertTrue(_is_whitespace("\n"      )      )
       self.assertTrue(_is_whitespace("\u00A0"      )      )

       self.assertFalse(_is_whitespace("A"      )      )
       self.assertFalse(_is_whitespace("-"      )      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       self.assertTrue(_is_control("\u0005"      )      )

       self.assertFalse(_is_control("A"      )      )
       self.assertFalse(_is_control(" "      )      )
       self.assertFalse(_is_control("\t"      )      )
       self.assertFalse(_is_control("\r"      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       self.assertTrue(_is_punctuation("-"      )      )
       self.assertTrue(_is_punctuation("$"      )      )
       self.assertTrue(_is_punctuation("`"      )      )
       self.assertTrue(_is_punctuation("."      )      )

       self.assertFalse(_is_punctuation("A"      )      )
       self.assertFalse(_is_punctuation(" "      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       lowercase_						:     Dict							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )

       self.assertListEqual(
           [rust_tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )




    @slow
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     str							     =  self.tokenizer_class.from_pretrained("bert-base-uncased"      )

       lowercase_						:     Tuple							     =  tokenizer.encode("sequence builders"	,  add_special_tokens=a      )
       lowercase_						:     Union[str, Any]							     =  tokenizer.encode("multi-sequence build"	,  add_special_tokens=a      )

       lowercase_						:     Dict							     =  tokenizer.build_inputs_with_special_tokens(a      )
       lowercase_						:     Optional[Any]							     =  tokenizer.build_inputs_with_special_tokens(a	,  a      )

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Dict							     =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
             lowercase_						:     int							     =  tokenizer_r.encode_plus(
                 a	,  return_attention_mask=a	,  return_token_type_ids=a	,  return_offsets_mapping=a	,  add_special_tokens=a	,  )

             lowercase_						:     Tuple							     =  tokenizer_r.do_lower_case if hasattr(a	,  "do_lower_case"      ) else False
             lowercase_						:     List[Any]							     =  (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "Allen"),
                     ((2_1, 2_3), "##NL"),
                     ((2_3, 2_4), "##P"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "allen"),
                     ((2_1, 2_3), "##nl"),
                     ((2_3, 2_4), "##p"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]	,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
             self.assertEqual([e[0] for e in expected_results]	,  tokens["offset_mapping"]      )





    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  ["的", "人", "有"]
       lowercase_						:     int							     =  "".join(a      )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  True
             lowercase_						:     Dict							     =  self.tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )

             lowercase_						:     str							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )

             lowercase_						:     Tuple							     =  False
             lowercase_						:     Union[str, Any]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )

             lowercase_						:     int							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     List[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase_						:     int							     =  [
                 f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a      )
             ]
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )




