




'''simple docstring'''







import copy
import json
import os
import tempfile

from transformers import is_torch_available

from .test_configuration_utils import config_common_kwargs


class 			_a  (	__a ):
       def __init__(	self						:	str   ,				lowercase						:	Union[str, Any]   ,				lowercase						:	Tuple=None   ,				lowercase						:	Any=True   ,				lowercase						:	Dict=None   ,				**lowercase						:	int				):



           '''simple docstring'''

           UpperCAmelCase			       =		parent
           UpperCAmelCase			       =		config_class
           UpperCAmelCase			       =		has_text_modality
           UpperCAmelCase			       =		kwargs
           UpperCAmelCase			       =		common_properties
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.config_class(**self.inputs_dict				)
           UpperCAmelCase			       =		(
               ['''hidden_size''', '''num_attention_heads''', '''num_hidden_layers''']
               if self.common_properties is None
               else self.common_properties
           )

           # Add common fields for text models
           if self.has_text_modality:
               common_properties.extend(['''vocab_size''']				)

           # Test that config has the common properties as getters
           for prop in common_properties:
               self.parent.assertTrue(hasattr(lowercase   ,				lowercase				)   ,				msg=f"`{prop}` does not exist"				)

           # Test that config has the common properties as setter
           for idx, name in enumerate(lowercase				):
               try:
                   setattr(lowercase   ,				lowercase   ,				lowercase				)
                   self.parent.assertEqual(
                       getattr(lowercase   ,				lowercase				)   ,				lowercase   ,				msg=f"`{name} value {idx} expected, but was {getattr(lowercase   ,				lowercase				)}"				)
               except NotImplementedError:
                   # Some models might not be able to implement setters for common_properties
                   # In that case, a NotImplementedError is raised
                   pass

        # Test if config class can be called with Config(prop_name=..)
           for idx, name in enumerate(lowercase				):
               try:
                   UpperCAmelCase			       =		self.config_class(**{name: idx}				)
                   self.parent.assertEqual(
                       getattr(lowercase   ,				lowercase				)   ,				lowercase   ,				msg=f"`{name} value {idx} expected, but was {getattr(lowercase   ,				lowercase				)}"				)
               except NotImplementedError:
                   # Some models might not be able to implement setters for common_properties
                   # In that case, a NotImplementedError is raised
                   pass
       def     A      (	self						:	List[Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.config_class(**self.inputs_dict				)
           UpperCAmelCase			       =		json.loads(config.to_json_string()				)
           for key, value in self.inputs_dict.items():
               self.parent.assertEqual(obj[key]   ,				lowercase				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.config_class(**self.inputs_dict				)

           with tempfile.TemporaryDirectory() as tmpdirname:
               UpperCAmelCase			       =		os.path.join(lowercase   ,				'''config.json'''				)
               config_first.to_json_file(lowercase				)
               UpperCAmelCase			       =		self.config_class.from_json_file(lowercase				)

           self.parent.assertEqual(config_second.to_dict()   ,				config_first.to_dict()				)
       def     A      (	self						:	List[Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.config_class(**self.inputs_dict				)

           with tempfile.TemporaryDirectory() as tmpdirname:
               config_first.save_pretrained(lowercase				)
               UpperCAmelCase			       =		self.config_class.from_pretrained(lowercase				)

           self.parent.assertEqual(config_second.to_dict()   ,				config_first.to_dict()				)
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.config_class(**self.inputs_dict				)

           UpperCAmelCase			       =		'''test'''
           with tempfile.TemporaryDirectory() as tmpdirname:
               UpperCAmelCase			       =		os.path.join(lowercase   ,				lowercase				)
               config_first.save_pretrained(lowercase				)
               UpperCAmelCase			       =		self.config_class.from_pretrained(lowercase   ,				subfolder=lowercase				)

           self.parent.assertEqual(config_second.to_dict()   ,				config_first.to_dict()				)
       def     A      (	self						:	Any				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.config_class(**self.inputs_dict   ,				num_labels=5				)
           self.parent.assertEqual(len(config.idalabel				)   ,				5				)
           self.parent.assertEqual(len(config.labelaid				)   ,				5				)

           UpperCAmelCase			       =		3
           self.parent.assertEqual(len(config.idalabel				)   ,				3				)
           self.parent.assertEqual(len(config.labelaid				)   ,				3				)
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           if self.config_class.is_composition:
               return
           UpperCAmelCase			       =		self.config_class()
           self.parent.assertIsNotNone(lowercase				)
       def     A      (	self						:	Optional[Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		copy.deepcopy(lowercase				)
           UpperCAmelCase			       =		self.config_class(**lowercase				)
           UpperCAmelCase			       =		[]
           for key, value in config_common_kwargs.items():
               if key == "torch_dtype":
                   if not is_torch_available():
                       continue
                   else:
                       import torch

                       if config.torch_dtype != torch.floataa:
                           wrong_values.append(('''torch_dtype''', config.torch_dtype, torch.floataa)				)
               elif getattr(lowercase   ,				lowercase				) != value:
                   wrong_values.append((key, getattr(lowercase   ,				lowercase				), value)				)

           if len(lowercase				) > 0:
               UpperCAmelCase			       =		'''\n'''.join([f"- {v[0]}: got {v[1]} instead of {v[2]}" for v in wrong_values]				)
               raise ValueError(f"The following keys were not properly set in the config:\n{errors}"				)







       def     A      (	self						:	Tuple				):



           '''simple docstring'''

           self.create_and_test_config_common_properties()
           self.create_and_test_config_to_json_string()
           self.create_and_test_config_to_json_file()
           self.create_and_test_config_from_and_save_pretrained()
           self.create_and_test_config_from_and_save_pretrained_subfolder()
           self.create_and_test_config_with_num_labels()
           self.check_config_can_be_init_without_params()
           self.check_config_arguments_init()











import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False
  a_        = True






  def 				lowercase		(				self :		Optional[Any]     )							->						List[str]:
     super().setUp()

     __lowerCAmelCase								=	[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         'こんにちは',
         'こん',
         'にちは',
         'ばんは',
         '##こん',
         '##にちは',
         '##ばんは',
         '世界',
         '##世界',
         '、',
         '##、',
         '。',
         '##。',
     ]

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		Tuple     )							->						str:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こんにちは 、 世界 。 こんばんは 、 世界 。'
     return input_text, output_text






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		str     )							->						Dict:
     __lowerCAmelCase	,	__lowerCAmelCase								=	self.get_input_output_texts(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode(lowerCAmelCase_					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.decode(lowerCAmelCase_					,  clean_up_tokenization_spaces=lowerCAmelCase_     )
     return text, ids






  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Optional[Any]     )							->						Optional[Any]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )






  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='mecab'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	MecabTokenizer(mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		List[Any]     )							->						int:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic_lite'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(do_lower_case=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(
            do_lower_case=lowerCAmelCase_					,  normalize_text=lowerCAmelCase_					,  mecab_option='-d /usr/local/lib/mecab/dic/jumandic'     )
     except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(normalize_text=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,  )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='sudachi'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='A'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国', '人', '参政', '権']     )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='B'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人', '参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						Union[str, Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='C'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		Dict     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(do_lower_case=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(normalize_text=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	SudachiTokenizer(trim_whitespace=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='jumanpp'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_jumanpp
  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	JumanppTokenizer(do_lower_case=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Dict     )							->						Dict:
     __lowerCAmelCase								=	JumanppTokenizer(normalize_text=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	JumanppTokenizer(trim_whitespace=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Any:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'     )					,  ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						str:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	WordpieceTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こんにちは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは'     )					,  ['こん', '##ばんは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'     )					,  ['こん', '##ばんは', '[UNK]', 'こんにちは']     )






  def 				lowercase		(				self :		List[Any]     )							->						Tuple:
     __lowerCAmelCase								=	BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'     )
     __lowerCAmelCase								=	tokenizer.subword_tokenizer

     __lowerCAmelCase								=	subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']     )

     __lowerCAmelCase								=	subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False






  def 				lowercase		(				self :		Optional[Any]     )							->						Tuple:
     super().setUp()

     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		str					,  **lowerCAmelCase_ :		Tuple     )							->						Union[str, Any]:
     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,  subword_tokenizer_type='character'					,  **lowerCAmelCase_     )






  def 				lowercase		(				self :		Tuple					,  lowerCAmelCase_ :		Tuple     )							->						Optional[int]:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
     return input_text, output_text






  def 				lowercase		(				self :		Dict     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Any     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		List[Any]     )							->						int:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		str     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  subword_tokenizer_type='character'     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'     )
     self.assertListEqual(
         lowerCAmelCase_					,  ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']     )
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]     )






  def 				lowercase		(				self :		str     )							->						Optional[int]:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	CharacterTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こ', 'ん', 'に', 'ち', 'は']     )

     self.assertListEqual(tokenizer.tokenize('こんにちほ'     )					,  ['こ', 'ん', 'に', 'ち', '[UNK]']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						Union[str, Any]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     __lowerCAmelCase								=	AutoTokenizer.from_pretrained(lowerCAmelCase_     )
     self.assertIsInstance(lowerCAmelCase_					,  lowerCAmelCase_     )





class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )
     __lowerCAmelCase								=	'bert-base-cased'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertJapaneseTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )

