import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin

class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Dict          =    ProphetNetTokenizer
    SCREAMING_SNAKE_CASE:				int          =    False




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       lowerCAmelCase_:							Union[str, Any]       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							Tuple       =	"UNwant\u00E9d,running"
       lowerCAmelCase_:							Any       =	"unwanted, running"
       return input_text, output_text




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	self.tokenizer_class(self.vocab_file )

       lowerCAmelCase_:							Optional[Any]       =	tokenizer.tokenize("UNwant\u00E9d,running" )
       self.assertListEqual(lowerCamelCase__       ,						["un", "##want", "##ed", ",", "runn", "##ing"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[9, 6, 7, 12, 10, 11] )




    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz" )       ,						["ah", "\u535A", "\u63A8", "zz"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	BasicTokenizer(do_lower_case=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )       ,						["hello", "!", "how", "are", "you", "?"] )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,						["hello"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	BasicTokenizer(do_lower_case=lowerCamelCase__       ,						strip_accents=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,						["hällo", "!", "how", "are", "you", "?"] )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,						["h\u00E9llo"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	BasicTokenizer(do_lower_case=lowerCamelCase__       ,						strip_accents=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,						["hallo", "!", "how", "are", "you", "?"] )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,						["hello"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	BasicTokenizer(do_lower_case=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,						["hallo", "!", "how", "are", "you", "?"] )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,						["hello"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	BasicTokenizer(do_lower_case=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )       ,						["HeLLo", "!", "how", "Are", "yoU", "?"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	BasicTokenizer(do_lower_case=lowerCamelCase__       ,						strip_accents=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,						["HäLLo", "!", "how", "Are", "yoU", "?"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	BasicTokenizer(do_lower_case=lowerCamelCase__       ,						strip_accents=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,						["HaLLo", "!", "how", "Are", "yoU", "?"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	BasicTokenizer(do_lower_case=lowerCamelCase__       ,						never_split=["[UNK]"] )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]" )       ,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       lowerCAmelCase_:							Optional[int]       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							Any       =	i
       lowerCAmelCase_:							Any       =	WordpieceTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("unwanted running" )       ,						["un", "##want", "##ed", "runn", "##ing"] )

       self.assertListEqual(tokenizer.tokenize("unwantedX running" )       ,						["[UNK]", "runn", "##ing"] )




    @require_torch
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased" )

       lowerCAmelCase_:							str       =	["A long paragraph for summarization.", "Another paragraph for summarization."]
       lowerCAmelCase_:							Optional[Any]       =	[1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
       lowerCAmelCase_:							Union[str, Any]       =	tokenizer(lowerCamelCase__       ,						padding=lowerCamelCase__       ,						return_tensors="pt" )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	list(batch.input_ids.numpy()[0] )
       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )

       self.assertEqual((2, 9)       ,						batch.input_ids.shape )
       self.assertEqual((2, 9)       ,						batch.attention_mask.shape )




    def      _a    (    self ):
       self.assertTrue(_is_whitespace(" " ) )
       self.assertTrue(_is_whitespace("\t" ) )
       self.assertTrue(_is_whitespace("\r" ) )
       self.assertTrue(_is_whitespace("\n" ) )
       self.assertTrue(_is_whitespace("\u00A0" ) )

       self.assertFalse(_is_whitespace("A" ) )
       self.assertFalse(_is_whitespace("-" ) )




    def      _a    (    self ):
       self.assertTrue(_is_control("\u0005" ) )

       self.assertFalse(_is_control("A" ) )
       self.assertFalse(_is_control(" " ) )
       self.assertFalse(_is_control("\t" ) )
       self.assertFalse(_is_control("\r" ) )




    def      _a    (    self ):
       self.assertTrue(_is_punctuation("-" ) )
       self.assertTrue(_is_punctuation("$" ) )
       self.assertTrue(_is_punctuation("`" ) )
       self.assertTrue(_is_punctuation("." ) )

       self.assertFalse(_is_punctuation("A" ) )
       self.assertFalse(_is_punctuation(" " ) )




    @slow
    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased" )

       lowerCAmelCase_:							Dict       =	tokenizer.encode("sequence builders"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Optional[int]       =	tokenizer.encode("multi-sequence build"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							int       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							Optional[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       assert encoded_sentence == text + [102]
       assert encoded_pair == text + [102] + text_a + [102]
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Union[str, Any]          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Dict          =    False
    SCREAMING_SNAKE_CASE:				List[Any]          =    True




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "こんにちは",
           "こん",
           "にちは",
           "ばんは",
           "##こん",
           "##にちは",
           "##ばんは",
           "世界",
           "##世界",
           "、",
           "##、",
           "。",
           "##。",
       ]

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							Optional[int]       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	"こんにちは 、 世界 。 こんばんは 、 世界 。"
       return input_text, output_text




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							str       =	self.get_input_output_texts(lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode(lowerCamelCase__       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							List[str]       =	tokenizer.decode(lowerCamelCase__       ,						clean_up_tokenization_spaces=lowerCamelCase__ )
       return text, ids




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	self.tokenizer_class(self.vocab_file )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。" )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="mecab" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[Any]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Dict       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Union[str, Any]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	MecabTokenizer(mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							List[Any]       =	MecabTokenizer(mecab_dic="unidic_lite" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							str       =	MecabTokenizer(mecab_dic="unidic" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	MecabTokenizer(do_lower_case=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							Any       =	MecabTokenizer(
              do_lower_case=lowerCamelCase__       ,						normalize_text=lowerCamelCase__       ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic" )
       except RuntimeError:
          # if dict doesn't exist in the system, previous code raises this error.
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	MecabTokenizer(normalize_text=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="sudachi" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[str]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[int]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							List[str]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Optional[int]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="A" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国", "人", "参政", "権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="B" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人", "参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="C" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	SudachiTokenizer(do_lower_case=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(normalize_text=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	SudachiTokenizer(trim_whitespace=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							int       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="jumanpp" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							Any       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Any       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(do_lower_case=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	JumanppTokenizer(normalize_text=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(trim_whitespace=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。" )       ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

       lowerCAmelCase_:							Tuple       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							List[Any]       =	i
       lowerCAmelCase_:							List[str]       =	WordpieceTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こんにちは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは" )       ,						["こん", "##ばんは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは" )       ,						["こん", "##ばんは", "[UNK]", "こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp" )
       lowerCAmelCase_:							Optional[Any]       =	tokenizer.subword_tokenizer

       lowerCAmelCase_:							List[str]       =	subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。" )
       self.assertListEqual(lowerCamelCase__       ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"] )

       lowerCAmelCase_:							Optional[int]       =	subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは" )
       self.assertListEqual(lowerCamelCase__       ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese" )

       lowerCAmelCase_:							int       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Optional[int]       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Tuple          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Optional[Any]          =    False




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							int       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						**lowerCamelCase__ ):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,						subword_tokenizer_type="character"       ,						**lowerCamelCase__ )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							str       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Tuple       =	"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
       return input_text, output_text




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class(self.vocab_file       ,						subword_tokenizer_type="character" )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。" )
       self.assertListEqual(
           lowerCamelCase__       ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"] )
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							List[str]       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							Optional[Any]       =	i
       lowerCAmelCase_:							List[Any]       =	CharacterTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こ", "ん", "に", "ち", "は"] )

       self.assertListEqual(tokenizer.tokenize("こんにちほ" )       ,						["こ", "ん", "に", "ち", "[UNK]"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char" )

       lowerCAmelCase_:							Any       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							Dict       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							Any       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	"cl-tohoku/bert-base-japanese"
       lowerCAmelCase_:							Optional[int]       =	AutoTokenizer.from_pretrained(lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )





class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	"cl-tohoku/bert-base-japanese"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )
       lowerCAmelCase_:							List[str]       =	"bert-base-cased"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertJapaneseTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )