



"""simple docstring"""






import argparse
from collections import defaultdict

import yaml


lowerCAmelCase__              =							'''docs/source/en/_toctree.yml'''






def 			snake_case_    ( A_   :						List[str]		):


       '''simple docstring'''





       _lowerCamelCase		:							int            =					defaultdict(A_		)
       for doc in model_doc:
              counts[doc["local"]] += 1
       _lowerCamelCase		:							List[Any]            =					[key for key, value in counts.items() if value > 1]

       _lowerCamelCase		:							List[str]            =					[]
       for duplicate_key in duplicates:
              _lowerCamelCase		:							Dict            =					list({doc['''title'''] for doc in model_doc if doc['''local'''] == duplicate_key}		)
              if len(A_		) > 1:
                     raise ValueError(
                         F'''{duplicate_key} is present several times in the documentation table of content at '''
                         '''`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '''
                         '''others.'''		)
              # Only add this once
              new_doc.append({'''local''': duplicate_key, '''title''': titles[0]}		)

       # Add none duplicate-keys
       new_doc.extend([doc for doc in model_doc if counts[doc['''local''']] == 1]		)

       # Sort
       return sorted(A_,							key=lambda A_		: s["title"].lower()		)






def 			snake_case_    ( A_   :						Union[str, Any]=False		):


       '''simple docstring'''





       with open(A_,							encoding='''utf-8'''		) as f:
              _lowerCamelCase		:							Dict            =					yaml.safe_load(f.read()		)

       # Get to the API doc
       _lowerCamelCase		:							List[str]            =					0
       while content[api_idx]["title"] != "API":
              api_idx += 1
       _lowerCamelCase		:							Any            =					content[api_idx]['''sections''']

       # Then to the model doc
       _lowerCamelCase		:							str            =					0
       while api_doc[model_idx]["title"] != "Models":
              model_idx += 1

       _lowerCamelCase		:							List[str]            =					api_doc[model_idx]['''sections''']

       _lowerCamelCase		:							Tuple            =					[(idx, section) for idx, section in enumerate(A_		) if '''sections''' in section]
       _lowerCamelCase		:							Any            =					False
       for idx, modality_doc in modalities_docs:
              _lowerCamelCase		:							str            =					modality_doc['''sections''']
              _lowerCamelCase		:							List[str]            =					clean_model_doc_toc(A_		)

              if old_modality_doc != new_modality_doc:
                     _lowerCamelCase		:							Any            =					True
                     if overwrite:
                            _lowerCamelCase		:							Optional[int]            =					new_modality_doc

       if diff:
              if overwrite:
                     _lowerCamelCase		:							Any            =					model_doc
                     _lowerCamelCase		:							Optional[Any]            =					api_doc
                     with open(A_,							'''w''',							encoding='''utf-8'''		) as f:
                            f.write(yaml.dump(A_,							allow_unicode=A_		)		)
              else:
                     raise ValueError(
                         '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''		)


if __name__ == "__main__":
   lowerCAmelCase__              =							argparse.ArgumentParser()
   parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
   lowerCAmelCase__              =							parser.parse_args()

   check_model_doc(args.fix_and_overwrite)



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Union[str, Any]          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Dict          =    False
    SCREAMING_SNAKE_CASE:				List[Any]          =    True




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "こんにちは",
           "こん",
           "にちは",
           "ばんは",
           "##こん",
           "##にちは",
           "##ばんは",
           "世界",
           "##世界",
           "、",
           "##、",
           "。",
           "##。",
       ]

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							Optional[int]       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	"こんにちは 、 世界 。 こんばんは 、 世界 。"
       return input_text, output_text




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							str       =	self.get_input_output_texts(lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode(lowerCamelCase__       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							List[str]       =	tokenizer.decode(lowerCamelCase__       ,						clean_up_tokenization_spaces=lowerCamelCase__ )
       return text, ids




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	self.tokenizer_class(self.vocab_file )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。" )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="mecab" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[Any]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Dict       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Union[str, Any]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	MecabTokenizer(mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							List[Any]       =	MecabTokenizer(mecab_dic="unidic_lite" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							str       =	MecabTokenizer(mecab_dic="unidic" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	MecabTokenizer(do_lower_case=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							Any       =	MecabTokenizer(
              do_lower_case=lowerCamelCase__       ,						normalize_text=lowerCamelCase__       ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic" )
       except RuntimeError:
          # if dict doesn't exist in the system, previous code raises this error.
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	MecabTokenizer(normalize_text=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="sudachi" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[str]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[int]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							List[str]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Optional[int]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="A" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国", "人", "参政", "権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="B" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人", "参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="C" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	SudachiTokenizer(do_lower_case=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(normalize_text=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	SudachiTokenizer(trim_whitespace=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							int       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="jumanpp" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							Any       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Any       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(do_lower_case=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	JumanppTokenizer(normalize_text=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(trim_whitespace=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。" )       ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

       lowerCAmelCase_:							Tuple       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							List[Any]       =	i
       lowerCAmelCase_:							List[str]       =	WordpieceTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こんにちは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは" )       ,						["こん", "##ばんは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは" )       ,						["こん", "##ばんは", "[UNK]", "こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp" )
       lowerCAmelCase_:							Optional[Any]       =	tokenizer.subword_tokenizer

       lowerCAmelCase_:							List[str]       =	subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。" )
       self.assertListEqual(lowerCamelCase__       ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"] )

       lowerCAmelCase_:							Optional[int]       =	subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは" )
       self.assertListEqual(lowerCamelCase__       ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese" )

       lowerCAmelCase_:							int       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Optional[int]       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Tuple          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Optional[Any]          =    False




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							int       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						**lowerCamelCase__ ):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,						subword_tokenizer_type="character"       ,						**lowerCamelCase__ )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							str       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Tuple       =	"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
       return input_text, output_text




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class(self.vocab_file       ,						subword_tokenizer_type="character" )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。" )
       self.assertListEqual(
           lowerCamelCase__       ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"] )
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							List[str]       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							Optional[Any]       =	i
       lowerCAmelCase_:							List[Any]       =	CharacterTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こ", "ん", "に", "ち", "は"] )

       self.assertListEqual(tokenizer.tokenize("こんにちほ" )       ,						["こ", "ん", "に", "ち", "[UNK]"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char" )

       lowerCAmelCase_:							Any       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							Dict       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							Any       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	"cl-tohoku/bert-base-japanese"
       lowerCAmelCase_:							Optional[int]       =	AutoTokenizer.from_pretrained(lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )





class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	"cl-tohoku/bert-base-japanese"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )
       lowerCAmelCase_:							List[str]       =	"bert-base-cased"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertJapaneseTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )