



"""simple docstring"""
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class       SCREAMING_SNAKE_CASE__  (					_UpperCamelCase      ,			unittest.TestCase ):
   __lowerCAmelCase  :   Union[str, Any]						=						BertJapaneseTokenizer
   __lowerCAmelCase  :   Tuple						=						False
   __lowerCAmelCase  :   Optional[Any]						=						True

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[str]:





    '''simple docstring'''




    super().setUp()

    UpperCAmelCase	:					str	=	[
        """[UNK]""",
        """[CLS]""",
        """[SEP]""",
        """こんにちは""",
        """こん""",
        """にちは""",
        """ばんは""",
        """##こん""",
        """##にちは""",
        """##ばんは""",
        """世界""",
        """##世界""",
        """、""",
        """##、""",
        """。""",
        """##。""",
    ]

    UpperCAmelCase	:					str	=	os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES["""vocab_file"""]						)
    with open(self.vocab_file      ,     """w"""      ,     encoding="""utf-8"""						) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE						)  ->      str:





    '''simple docstring'''




    UpperCAmelCase	:					int	=	"""こんにちは、世界。 \nこんばんは、世界。"""
    UpperCAmelCase	:					str	=	"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
    return input_text, output_text

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE						)  ->      Dict:





    '''simple docstring'''




    UpperCAmelCase	,		UpperCAmelCase	:					Optional[Any]	=	self.get_input_output_texts(lowerCAmelCase_						)
    UpperCAmelCase	:					Optional[Any]	=	tokenizer.encode(lowerCAmelCase_      ,     add_special_tokens=lowerCAmelCase_						)
    UpperCAmelCase	:					str	=	tokenizer.decode(lowerCAmelCase_      ,     clean_up_tokenization_spaces=lowerCAmelCase_						)
    return text, ids

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[int]:





    '''simple docstring'''




    pass  # TODO add if relevant

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[Any]:





    '''simple docstring'''




    pass  # TODO add if relevant

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Any:





    '''simple docstring'''




    pass  # TODO add if relevant

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[Any]	=	self.tokenizer_class(self.vocab_file						)

    UpperCAmelCase	:					Optional[Any]	=	tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""						)
    self.assertListEqual(lowerCAmelCase_      ,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)      ,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[str]:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	self.tokenizer_class(self.vocab_file      ,     word_tokenizer_type="""mecab"""						)
    self.assertIsNotNone(lowerCAmelCase_						)

    UpperCAmelCase	:					Optional[Any]	=	"""こんにちは、世界。\nこんばんは、世界。"""
    UpperCAmelCase	:					Union[str, Any]	=	tokenizer.tokenize(lowerCAmelCase_						)
    self.assertListEqual(lowerCAmelCase_      ,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)      ,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

    UpperCAmelCase	:					Any	=	os.path.join(self.tmpdirname      ,     """tokenizer.bin"""						)
    with open(lowerCAmelCase_      ,     """wb"""						) as handle:
     pickle.dump(lowerCAmelCase_      ,     lowerCAmelCase_						)

    with open(lowerCAmelCase_      ,     """rb"""						) as handle:
     UpperCAmelCase	:					Optional[Any]	=	pickle.load(lowerCAmelCase_						)

    UpperCAmelCase	:					List[Any]	=	tokenizer_new.tokenize(lowerCAmelCase_						)

    self.assertListEqual(lowerCAmelCase_      ,     lowerCAmelCase_						)

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					Any	=	MecabTokenizer(mecab_dic="""ipadic"""						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,     )

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      int:





    '''simple docstring'''




    try:
     UpperCAmelCase	:					Tuple	=	MecabTokenizer(mecab_dic="""unidic_lite"""						)
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,     )

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[Any]:





    '''simple docstring'''




    try:
     UpperCAmelCase	:					Dict	=	MecabTokenizer(mecab_dic="""unidic"""						)
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,     )

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Union[str, Any]	=	MecabTokenizer(do_lower_case=lowerCAmelCase_      ,     mecab_dic="""ipadic"""						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,     )

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[Any]:





    '''simple docstring'''




    try:
     UpperCAmelCase	:					List[str]	=	MecabTokenizer(
         do_lower_case=lowerCAmelCase_      ,     normalize_text=lowerCAmelCase_      ,     mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""						)
    except RuntimeError:
     # if dict doesn't exist in the system, previous code raises this error.
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,     )

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Union[str, Any]	=	MecabTokenizer(normalize_text=lowerCAmelCase_      ,     mecab_dic="""ipadic"""						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]      ,     )

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[str]:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	self.tokenizer_class(self.vocab_file      ,     word_tokenizer_type="""sudachi"""						)
    self.assertIsNotNone(lowerCAmelCase_						)

    UpperCAmelCase	:					Any	=	"""こんにちは、世界。\nこんばんは、世界。"""
    UpperCAmelCase	:					Dict	=	tokenizer.tokenize(lowerCAmelCase_						)
    self.assertListEqual(lowerCAmelCase_      ,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)      ,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

    UpperCAmelCase	:					Dict	=	os.path.join(self.tmpdirname      ,     """tokenizer.bin"""						)
    with open(lowerCAmelCase_      ,     """wb"""						) as handle:
     pickle.dump(lowerCAmelCase_      ,     lowerCAmelCase_						)

    with open(lowerCAmelCase_      ,     """rb"""						) as handle:
     UpperCAmelCase	:					Optional[Any]	=	pickle.load(lowerCAmelCase_						)

    UpperCAmelCase	:					List[str]	=	tokenizer_new.tokenize(lowerCAmelCase_						)

    self.assertListEqual(lowerCAmelCase_      ,     lowerCAmelCase_						)

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[str]:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	SudachiTokenizer(sudachi_dict_type="""core"""						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     [""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,     )

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	SudachiTokenizer(sudachi_dict_type="""core"""      ,     sudachi_split_mode="""A"""						)

    self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)      ,     ["""外国""", """人""", """参政""", """権"""]						)

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	SudachiTokenizer(sudachi_dict_type="""core"""      ,     sudachi_split_mode="""B"""						)

    self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)      ,     ["""外国人""", """参政権"""]						)

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Any	=	SudachiTokenizer(sudachi_dict_type="""core"""      ,     sudachi_split_mode="""C"""						)

    self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)      ,     ["""外国人参政権"""]						)

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[str]:





    '''simple docstring'''




    UpperCAmelCase	:					Any	=	SudachiTokenizer(do_lower_case=lowerCAmelCase_      ,     sudachi_dict_type="""core"""						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     [""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,     )

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[Any]:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	SudachiTokenizer(normalize_text=lowerCAmelCase_      ,     sudachi_dict_type="""core"""						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     [""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]      ,     )

   @require_sudachi
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[int]	=	SudachiTokenizer(trim_whitespace=lowerCAmelCase_      ,     sudachi_dict_type="""core"""						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,     )

   @require_jumanpp
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Any:





    '''simple docstring'''




    UpperCAmelCase	:					Tuple	=	self.tokenizer_class(self.vocab_file      ,     word_tokenizer_type="""jumanpp"""						)
    self.assertIsNotNone(lowerCAmelCase_						)

    UpperCAmelCase	:					Tuple	=	"""こんにちは、世界。\nこんばんは、世界。"""
    UpperCAmelCase	:					Optional[int]	=	tokenizer.tokenize(lowerCAmelCase_						)
    self.assertListEqual(lowerCAmelCase_      ,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)      ,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

    UpperCAmelCase	:					Any	=	os.path.join(self.tmpdirname      ,     """tokenizer.bin"""						)
    with open(lowerCAmelCase_      ,     """wb"""						) as handle:
     pickle.dump(lowerCAmelCase_      ,     lowerCAmelCase_						)

    with open(lowerCAmelCase_      ,     """rb"""						) as handle:
     UpperCAmelCase	:					Dict	=	pickle.load(lowerCAmelCase_						)

    UpperCAmelCase	:					int	=	tokenizer_new.tokenize(lowerCAmelCase_						)

    self.assertListEqual(lowerCAmelCase_      ,     lowerCAmelCase_						)

   @require_jumanpp
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,     )

   @require_jumanpp
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	JumanppTokenizer(do_lower_case=lowerCAmelCase_						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,     )

   @require_jumanpp
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Dict:





    '''simple docstring'''




    UpperCAmelCase	:					Any	=	JumanppTokenizer(normalize_text=lowerCAmelCase_						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,     )

   @require_jumanpp
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      List[str]:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[Any]	=	JumanppTokenizer(trim_whitespace=lowerCAmelCase_						)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]      ,     )

   @require_jumanpp
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Any:





    '''simple docstring'''




    UpperCAmelCase	:					List[Any]	=	JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""						)      ,     ["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]      ,     )

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[int]	=	["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

    UpperCAmelCase	:					int	=	{}
    for i, token in enumerate(lowerCAmelCase_						):
     UpperCAmelCase	:					Union[str, Any]	=	i
    UpperCAmelCase	:					Any	=	WordpieceTokenizer(vocab=lowerCAmelCase_      ,     unk_token="""[UNK]"""						)

    self.assertListEqual(tokenizer.tokenize(""""""						)      ,     []						)

    self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)      ,     ["""こんにちは"""]						)

    self.assertListEqual(tokenizer.tokenize("""こんばんは"""						)      ,     ["""こん""", """##ばんは"""]						)

    self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""						)      ,     ["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]						)

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""						)
    UpperCAmelCase	:					Any	=	tokenizer.subword_tokenizer

    UpperCAmelCase	:					List[str]	=	subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""						)
    self.assertListEqual(lowerCAmelCase_      ,     ["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]						)

    UpperCAmelCase	:					Optional[int]	=	subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""						)
    self.assertListEqual(lowerCAmelCase_      ,     ["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]						)




   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""						)

    UpperCAmelCase	:					Optional[int]	=	tokenizer.encode("""ありがとう。"""      ,     add_special_tokens=lowerCAmelCase_						)
    UpperCAmelCase	:					int	=	tokenizer.encode("""どういたしまして。"""      ,     add_special_tokens=lowerCAmelCase_						)

    UpperCAmelCase	:					int	=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
    UpperCAmelCase	:					List[Any]	=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_      ,     lowerCAmelCase_						)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class       SCREAMING_SNAKE_CASE__  (					_UpperCamelCase      ,			unittest.TestCase ):
   __lowerCAmelCase  :   Dict						=						BertJapaneseTokenizer
   __lowerCAmelCase  :   Optional[int]						=						False

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Tuple:





    '''simple docstring'''




    super().setUp()

    UpperCAmelCase	:					Tuple	=	["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

    UpperCAmelCase	:					Optional[int]	=	os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES["""vocab_file"""]						)
    with open(self.vocab_file      ,     """w"""      ,     encoding="""utf-8"""						) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)

   def 	SCREAMING_SNAKE_CASE     (   self      ,     **_SCREAMING_SNAKE_CASE						)  ->      Union[str, Any]:





    '''simple docstring'''




    return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,     subword_tokenizer_type="""character"""      ,     **lowerCAmelCase_						)

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE						)  ->      Optional[int]:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	"""こんにちは、世界。 \nこんばんは、世界。"""
    UpperCAmelCase	:					List[Any]	=	"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
    return input_text, output_text

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    pass  # TODO add if relevant

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    pass  # TODO add if relevant

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      int:





    '''simple docstring'''




    pass  # TODO add if relevant

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	self.tokenizer_class(self.vocab_file      ,     subword_tokenizer_type="""character"""						)

    UpperCAmelCase	:					int	=	tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""						)
    self.assertListEqual(
        lowerCAmelCase_      ,     ["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]						)
    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)      ,     [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[int]:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[int]	=	["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

    UpperCAmelCase	:					Any	=	{}
    for i, token in enumerate(lowerCAmelCase_						):
     UpperCAmelCase	:					Optional[Any]	=	i
    UpperCAmelCase	:					Tuple	=	CharacterTokenizer(vocab=lowerCAmelCase_      ,     unk_token="""[UNK]"""						)

    self.assertListEqual(tokenizer.tokenize(""""""						)      ,     []						)

    self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)      ,     ["""こ""", """ん""", """に""", """ち""", """は"""]						)

    self.assertListEqual(tokenizer.tokenize("""こんにちほ"""						)      ,     ["""こ""", """ん""", """に""", """ち""", """[UNK]"""]						)




   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    UpperCAmelCase	:					Union[str, Any]	=	self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""						)

    UpperCAmelCase	:					int	=	tokenizer.encode("""ありがとう。"""      ,     add_special_tokens=lowerCAmelCase_						)
    UpperCAmelCase	:					List[Any]	=	tokenizer.encode("""どういたしまして。"""      ,     add_special_tokens=lowerCAmelCase_						)

    UpperCAmelCase	:					Dict	=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
    UpperCAmelCase	:					Union[str, Any]	=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_      ,     lowerCAmelCase_						)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class       SCREAMING_SNAKE_CASE__  (					unittest.TestCase ):

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	"""cl-tohoku/bert-base-japanese"""
    UpperCAmelCase	:					Dict	=	AutoTokenizer.from_pretrained(lowerCAmelCase_						)
    self.assertIsInstance(lowerCAmelCase_      ,     lowerCAmelCase_						)



class       SCREAMING_SNAKE_CASE__  (					unittest.TestCase ):

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[int]:





    '''simple docstring'''




    UpperCAmelCase	:					List[Any]	=	"""cl-tohoku/bert-base-japanese"""
    with self.assertLogs("""transformers"""      ,     level="""WARNING"""						) as cm:
     BertTokenizer.from_pretrained(lowerCAmelCase_						)
     self.assertTrue(
         cm.records[0].message.startswith(
             """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
             """ is called from."""						)						)
    UpperCAmelCase	:					Optional[Any]	=	"""bert-base-cased"""
    with self.assertLogs("""transformers"""      ,     level="""WARNING"""						) as cm:
     BertJapaneseTokenizer.from_pretrained(lowerCAmelCase_						)
     self.assertTrue(
         cm.records[0].message.startswith(
             """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
             """ is called from."""						)						)







from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_snake_case  :    List[Any]						    =  logging.get_logger(__name__)

_snake_case  :    List[Any]						    =  {
    'microsoft/beit-base-patch16-224-pt22k': (
        'https://huggingface.co/microsoft/beit-base-patch16-224-pt22k/resolve/main/config.json'
    ),
    # See all BEiT models at https://huggingface.co/models?filter=beit
}



class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  a_        = """beit"""






  def __init__(				self :		List[Any]					,  lowerCAmelCase_ :		Tuple=8_1_9_2					,  lowerCAmelCase_ :		Optional[int]=7_6_8					,  lowerCAmelCase_ :		int=1_2					,  lowerCAmelCase_ :		Optional[int]=1_2					,  lowerCAmelCase_ :		Any=3_0_7_2					,  lowerCAmelCase_ :		Optional[int]="gelu"					,  lowerCAmelCase_ :		Any=0.0					,  lowerCAmelCase_ :		Any=0.0					,  lowerCAmelCase_ :		Any=0.02					,  lowerCAmelCase_ :		int=1e-12					,  lowerCAmelCase_ :		int=2_2_4					,  lowerCAmelCase_ :		str=1_6					,  lowerCAmelCase_ :		int=3					,  lowerCAmelCase_ :		Dict=False					,  lowerCAmelCase_ :		int=False					,  lowerCAmelCase_ :		List[Any]=False					,  lowerCAmelCase_ :		int=False					,  lowerCAmelCase_ :		List[str]=0.1					,  lowerCAmelCase_ :		Union[str, Any]=0.1					,  lowerCAmelCase_ :		List[str]=True					,  lowerCAmelCase_ :		List[Any]=[3, 5, 7, 1_1]					,  lowerCAmelCase_ :		Optional[Any]=[1, 2, 3, 6]					,  lowerCAmelCase_ :		Tuple=True					,  lowerCAmelCase_ :		Dict=0.4					,  lowerCAmelCase_ :		Tuple=2_5_6					,  lowerCAmelCase_ :		Any=1					,  lowerCAmelCase_ :		Any=False					,  lowerCAmelCase_ :		Optional[int]=2_5_5					,  **lowerCAmelCase_ :		Any					,  )							->						Dict:
     super().__init__(**lowerCAmelCase_     )

     __lowerCAmelCase								=	vocab_size
     __lowerCAmelCase								=	hidden_size
     __lowerCAmelCase								=	num_hidden_layers
     __lowerCAmelCase								=	num_attention_heads
     __lowerCAmelCase								=	intermediate_size
     __lowerCAmelCase								=	hidden_act
     __lowerCAmelCase								=	hidden_dropout_prob
     __lowerCAmelCase								=	attention_probs_dropout_prob
     __lowerCAmelCase								=	initializer_range
     __lowerCAmelCase								=	layer_norm_eps

     __lowerCAmelCase								=	image_size
     __lowerCAmelCase								=	patch_size
     __lowerCAmelCase								=	num_channels
     __lowerCAmelCase								=	use_mask_token
     __lowerCAmelCase								=	use_absolute_position_embeddings
     __lowerCAmelCase								=	use_relative_position_bias
     __lowerCAmelCase								=	use_shared_relative_position_bias
     __lowerCAmelCase								=	layer_scale_init_value
     __lowerCAmelCase								=	drop_path_rate
     __lowerCAmelCase								=	use_mean_pooling
     # decode head attributes (semantic segmentation)
     __lowerCAmelCase								=	out_indices
     __lowerCAmelCase								=	pool_scales
     # auxiliary head attributes (semantic segmentation)
     __lowerCAmelCase								=	use_auxiliary_head
     __lowerCAmelCase								=	auxiliary_loss_weight
     __lowerCAmelCase								=	auxiliary_channels
     __lowerCAmelCase								=	auxiliary_num_convs
     __lowerCAmelCase								=	auxiliary_concat_input
     __lowerCAmelCase								=	semantic_loss_ignore_index





class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  a_        = version.parse("""1.11"""    )






  @property
  def 				lowercase		(				self :		Tuple     )							->						Mapping[str, Mapping[int, str]]:
     return OrderedDict(
         [
             ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
         ]     )






  @property
  def 				lowercase		(				self :		Optional[Any]     )							->						float:
     return 1e-4

