

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 	snake_case_   (		__A	,       unittest.TestCase					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   List[str]									=				BertJapaneseTokenizer
			SCREAMING_SNAKE_CASE     :   Dict									=				False
			SCREAMING_SNAKE_CASE     :   str									=				True







			def 				snake_case__(     self :				str							)      ->Tuple:
									super().setUp()

									snake_case_   =	[
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''こんにちは''',
									    '''こん''',
									    '''にちは''',
									    '''ばんは''',
									    '''##こん''',
									    '''##にちは''',
									    '''##ばんは''',
									    '''世界''',
									    '''##世界''',
									    '''、''',
									    '''##、''',
									    '''。''',
									    '''##。''',
									]

									snake_case_   =	os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['''vocab_file''']							)
									with open(self.vocab_file							,   '''w'''							,   encoding='''utf-8'''							) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)







			def 				snake_case__(     self :				Optional[Any]							,   _UpperCamelCase :				List[Any]							)      ->List[str]:
									snake_case_   =	'''こんにちは、世界。 \nこんばんは、世界。'''
									snake_case_   =	'''こんにちは 、 世界 。 こんばんは 、 世界 。'''
									return input_text, output_text







			def 				snake_case__(     self :				Optional[Any]							,   _UpperCamelCase :				Dict							)      ->Tuple:
									snake_case_,    snake_case_   =	self.get_input_output_texts(_UpperCamelCase							)
									snake_case_   =	tokenizer.encode(_UpperCamelCase							,   add_special_tokens=_UpperCamelCase							)
									snake_case_   =	tokenizer.decode(_UpperCamelCase							,   clean_up_tokenization_spaces=_UpperCamelCase							)
									return text, ids







			def 				snake_case__(     self :				Any							)      ->Dict:
									pass  # TODO add if relevant







			def 				snake_case__(     self :				Optional[Any]							)      ->Optional[Any]:
									pass  # TODO add if relevant







			def 				snake_case__(     self :				Optional[Any]							)      ->Any:
									pass  # TODO add if relevant







			def 				snake_case__(     self :				Optional[int]							)      ->int:
									snake_case_   =	self.tokenizer_class(self.vocab_file							)

									snake_case_   =	tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''							)
									self.assertListEqual(_UpperCamelCase							,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase							)							,   [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)







			def 				snake_case__(     self :				Dict							)      ->Any:
									snake_case_   =	self.tokenizer_class(self.vocab_file							,   word_tokenizer_type='''mecab'''							)
									self.assertIsNotNone(_UpperCamelCase							)

									snake_case_   =	'''こんにちは、世界。\nこんばんは、世界。'''
									snake_case_   =	tokenizer.tokenize(_UpperCamelCase							)
									self.assertListEqual(_UpperCamelCase							,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase							)							,   [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

									snake_case_   =	os.path.join(self.tmpdirname							,   '''tokenizer.bin'''							)
									with open(_UpperCamelCase							,   '''wb'''							) as handle:
															pickle.dump(_UpperCamelCase							,   _UpperCamelCase							)

									with open(_UpperCamelCase							,   '''rb'''							) as handle:
															snake_case_   =	pickle.load(_UpperCamelCase							)

									snake_case_   =	tokenizer_new.tokenize(_UpperCamelCase							)

									self.assertListEqual(_UpperCamelCase							,   _UpperCamelCase							)







			def 				snake_case__(     self :				List[Any]							)      ->Tuple:
									snake_case_   =	MecabTokenizer(mecab_dic='''ipadic'''							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']							,   )







			def 				snake_case__(     self :				int							)      ->List[Any]:
									try:
															snake_case_   =	MecabTokenizer(mecab_dic='''unidic_lite'''							)
									except ModuleNotFoundError:
															return

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']							,   )







			def 				snake_case__(     self :				Union[str, Any]							)      ->str:
									try:
															snake_case_   =	MecabTokenizer(mecab_dic='''unidic'''							)
									except ModuleNotFoundError:
															return

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']							,   )







			def 				snake_case__(     self :				List[str]							)      ->Dict:
									snake_case_   =	MecabTokenizer(do_lower_case=_UpperCamelCase							,   mecab_dic='''ipadic'''							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']							,   )







			def 				snake_case__(     self :				Optional[int]							)      ->List[str]:
									try:
															snake_case_   =	MecabTokenizer(
															    do_lower_case=_UpperCamelCase							,   normalize_text=_UpperCamelCase							,   mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''							)
									except RuntimeError:
															# if dict doesn't exist in the system, previous code raises this error.
															return

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']							,   )







			def 				snake_case__(     self :				Optional[int]							)      ->Union[str, Any]:
									snake_case_   =	MecabTokenizer(normalize_text=_UpperCamelCase							,   mecab_dic='''ipadic'''							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']							,   )







			@require_sudachi
			def 				snake_case__(     self :				Optional[Any]							)      ->str:
									snake_case_   =	self.tokenizer_class(self.vocab_file							,   word_tokenizer_type='''sudachi'''							)
									self.assertIsNotNone(_UpperCamelCase							)

									snake_case_   =	'''こんにちは、世界。\nこんばんは、世界。'''
									snake_case_   =	tokenizer.tokenize(_UpperCamelCase							)
									self.assertListEqual(_UpperCamelCase							,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase							)							,   [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

									snake_case_   =	os.path.join(self.tmpdirname							,   '''tokenizer.bin'''							)
									with open(_UpperCamelCase							,   '''wb'''							) as handle:
															pickle.dump(_UpperCamelCase							,   _UpperCamelCase							)

									with open(_UpperCamelCase							,   '''rb'''							) as handle:
															snake_case_   =	pickle.load(_UpperCamelCase							)

									snake_case_   =	tokenizer_new.tokenize(_UpperCamelCase							)

									self.assertListEqual(_UpperCamelCase							,   _UpperCamelCase							)







			@require_sudachi
			def 				snake_case__(     self :				Tuple							)      ->Optional[int]:
									snake_case_   =	SudachiTokenizer(sudachi_dict_type='''core'''							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']							,   )







			@require_sudachi
			def 				snake_case__(     self :				str							)      ->Tuple:
									snake_case_   =	SudachiTokenizer(sudachi_dict_type='''core'''							,   sudachi_split_mode='''A'''							)

									self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							)							,   ['''外国''', '''人''', '''参政''', '''権''']							)







			@require_sudachi
			def 				snake_case__(     self :				Dict							)      ->List[Any]:
									snake_case_   =	SudachiTokenizer(sudachi_dict_type='''core'''							,   sudachi_split_mode='''B'''							)

									self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							)							,   ['''外国人''', '''参政権''']							)







			@require_sudachi
			def 				snake_case__(     self :				Optional[int]							)      ->Tuple:
									snake_case_   =	SudachiTokenizer(sudachi_dict_type='''core'''							,   sudachi_split_mode='''C'''							)

									self.assertListEqual(tokenizer.tokenize('''外国人参政権'''							)							,   ['''外国人参政権''']							)







			@require_sudachi
			def 				snake_case__(     self :				Optional[Any]							)      ->int:
									snake_case_   =	SudachiTokenizer(do_lower_case=_UpperCamelCase							,   sudachi_dict_type='''core'''							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']							,   )







			@require_sudachi
			def 				snake_case__(     self :				Dict							)      ->List[str]:
									snake_case_   =	SudachiTokenizer(normalize_text=_UpperCamelCase							,   sudachi_dict_type='''core'''							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']							,   )







			@require_sudachi
			def 				snake_case__(     self :				List[str]							)      ->List[Any]:
									snake_case_   =	SudachiTokenizer(trim_whitespace=_UpperCamelCase							,   sudachi_dict_type='''core'''							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']							,   )







			@require_jumanpp
			def 				snake_case__(     self :				int							)      ->Union[str, Any]:
									snake_case_   =	self.tokenizer_class(self.vocab_file							,   word_tokenizer_type='''jumanpp'''							)
									self.assertIsNotNone(_UpperCamelCase							)

									snake_case_   =	'''こんにちは、世界。\nこんばんは、世界。'''
									snake_case_   =	tokenizer.tokenize(_UpperCamelCase							)
									self.assertListEqual(_UpperCamelCase							,   ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']							)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase							)							,   [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

									snake_case_   =	os.path.join(self.tmpdirname							,   '''tokenizer.bin'''							)
									with open(_UpperCamelCase							,   '''wb'''							) as handle:
															pickle.dump(_UpperCamelCase							,   _UpperCamelCase							)

									with open(_UpperCamelCase							,   '''rb'''							) as handle:
															snake_case_   =	pickle.load(_UpperCamelCase							)

									snake_case_   =	tokenizer_new.tokenize(_UpperCamelCase							)

									self.assertListEqual(_UpperCamelCase							,   _UpperCamelCase							)







			@require_jumanpp
			def 				snake_case__(     self :				List[str]							)      ->Dict:
									snake_case_   =	JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']							,   )







			@require_jumanpp
			def 				snake_case__(     self :				Any							)      ->Any:
									snake_case_   =	JumanppTokenizer(do_lower_case=_UpperCamelCase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']							,   )







			@require_jumanpp
			def 				snake_case__(     self :				int							)      ->Dict:
									snake_case_   =	JumanppTokenizer(normalize_text=_UpperCamelCase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']							,   )







			@require_jumanpp
			def 				snake_case__(     self :				int							)      ->Optional[Any]:
									snake_case_   =	JumanppTokenizer(trim_whitespace=_UpperCamelCase							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''							)							,   ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']							,   )







			@require_jumanpp
			def 				snake_case__(     self :				Any							)      ->Optional[int]:
									snake_case_   =	JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''							)							,   ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']							,   )







			def 				snake_case__(     self :				Any							)      ->List[Any]:
									snake_case_   =	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

									snake_case_   =	{}
									for i, token in enumerate(_UpperCamelCase							):
															snake_case_   =	i
									snake_case_   =	WordpieceTokenizer(vocab=_UpperCamelCase							,   unk_token='''[UNK]'''							)

									self.assertListEqual(tokenizer.tokenize(''''''							)							,   []							)

									self.assertListEqual(tokenizer.tokenize('''こんにちは'''							)							,   ['''こんにちは''']							)

									self.assertListEqual(tokenizer.tokenize('''こんばんは'''							)							,   ['''こん''', '''##ばんは''']							)

									self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''							)							,   ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']							)







			def 				snake_case__(     self :				Optional[Any]							)      ->Optional[int]:
									snake_case_   =	BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''							)
									snake_case_   =	tokenizer.subword_tokenizer

									snake_case_   =	subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''							)
									self.assertListEqual(_UpperCamelCase							,   ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']							)

									snake_case_   =	subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''							)
									self.assertListEqual(_UpperCamelCase							,   ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']							)







			def 				snake_case__(     self :				str							)      ->Tuple:
									snake_case_   =	self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''							)

									snake_case_   =	tokenizer.encode('''ありがとう。'''							,   add_special_tokens=_UpperCamelCase							)
									snake_case_   =	tokenizer.encode('''どういたしまして。'''							,   add_special_tokens=_UpperCamelCase							)

									snake_case_   =	tokenizer.build_inputs_with_special_tokens(_UpperCamelCase							)
									snake_case_   =	tokenizer.build_inputs_with_special_tokens(_UpperCamelCase							,   _UpperCamelCase							)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 	snake_case_   (		__A	,       unittest.TestCase					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Union[str, Any]									=				BertJapaneseTokenizer
			SCREAMING_SNAKE_CASE     :   int									=				False







			def 				snake_case__(     self :				List[str]							)      ->int:
									super().setUp()

									snake_case_   =	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

									snake_case_   =	os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['''vocab_file''']							)
									with open(self.vocab_file							,   '''w'''							,   encoding='''utf-8'''							) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)







			def 				snake_case__(     self :				Optional[Any]							,   **_UpperCamelCase :				Union[str, Any]							)      ->int:
									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,   subword_tokenizer_type='''character'''							,   **_UpperCamelCase							)







			def 				snake_case__(     self :				Any							,   _UpperCamelCase :				Union[str, Any]							)      ->List[Any]:
									snake_case_   =	'''こんにちは、世界。 \nこんばんは、世界。'''
									snake_case_   =	'''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
									return input_text, output_text







			def 				snake_case__(     self :				Dict							)      ->Union[str, Any]:
									pass  # TODO add if relevant







			def 				snake_case__(     self :				Any							)      ->Union[str, Any]:
									pass  # TODO add if relevant







			def 				snake_case__(     self :				Tuple							)      ->Tuple:
									pass  # TODO add if relevant







			def 				snake_case__(     self :				List[Any]							)      ->int:
									snake_case_   =	self.tokenizer_class(self.vocab_file							,   subword_tokenizer_type='''character'''							)

									snake_case_   =	tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''							)
									self.assertListEqual(
									    _UpperCamelCase							,   ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']							)
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(_UpperCamelCase							)							,   [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)







			def 				snake_case__(     self :				List[str]							)      ->List[str]:
									snake_case_   =	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

									snake_case_   =	{}
									for i, token in enumerate(_UpperCamelCase							):
															snake_case_   =	i
									snake_case_   =	CharacterTokenizer(vocab=_UpperCamelCase							,   unk_token='''[UNK]'''							)

									self.assertListEqual(tokenizer.tokenize(''''''							)							,   []							)

									self.assertListEqual(tokenizer.tokenize('''こんにちは'''							)							,   ['''こ''', '''ん''', '''に''', '''ち''', '''は''']							)

									self.assertListEqual(tokenizer.tokenize('''こんにちほ'''							)							,   ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']							)







			def 				snake_case__(     self :				Dict							)      ->Tuple:
									snake_case_   =	self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''							)

									snake_case_   =	tokenizer.encode('''ありがとう。'''							,   add_special_tokens=_UpperCamelCase							)
									snake_case_   =	tokenizer.encode('''どういたしまして。'''							,   add_special_tokens=_UpperCamelCase							)

									snake_case_   =	tokenizer.build_inputs_with_special_tokens(_UpperCamelCase							)
									snake_case_   =	tokenizer.build_inputs_with_special_tokens(_UpperCamelCase							,   _UpperCamelCase							)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 	snake_case_   (		unittest.TestCase					):







			'''simple docstring'''

			def 				snake_case__(     self :				str							)      ->int:
									snake_case_   =	'''cl-tohoku/bert-base-japanese'''
									snake_case_   =	AutoTokenizer.from_pretrained(_UpperCamelCase							)
									self.assertIsInstance(_UpperCamelCase							,   _UpperCamelCase							)






class 	snake_case_   (		unittest.TestCase					):







			'''simple docstring'''

			def 				snake_case__(     self :				Optional[int]							)      ->Dict:
									snake_case_   =	'''cl-tohoku/bert-base-japanese'''
									with self.assertLogs('''transformers'''							,   level='''WARNING'''							) as cm:
															BertTokenizer.from_pretrained(_UpperCamelCase							)
															self.assertTrue(
															    cm.records[0].message.startswith(
															        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
															        ''' is called from.'''							)							)
									snake_case_   =	'''bert-base-cased'''
									with self.assertLogs('''transformers'''							,   level='''WARNING'''							) as cm:
															BertJapaneseTokenizer.from_pretrained(_UpperCamelCase							)
															self.assertTrue(
															    cm.records[0].message.startswith(
															        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
															        ''' is called from.'''							)							)




'''simple docstring'''




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
a   :			Tuple					  =    {"""LayoutLMv2Config""", """LayoutLMv3Config"""}

@is_pipeline_test
class 			UpperCamelCase_ (							unittest.TestCase			):
	lowercase          = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
	lowercase          = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

	if model_mapping is not None:
		lowercase          = {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
	if tf_model_mapping is not None:
		lowercase          = {
		    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
		}





	def  _lowercase(  self		,		A		,		A		,		A    )			->			Dict:
								UpperCAmelCase     :		Union[str, Any]							     =		ZeroShotClassificationPipeline(
								    model=A		,		tokenizer=A		,		candidate_labels=["""polics""", """health"""]    )
								return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]





	def  _lowercase(  self		,		A		,		A    )			->			Optional[int]:
								UpperCAmelCase     :		Dict							     =		classifier("""Who are you voting for in 2020?"""		,		candidate_labels="""politics"""    )
								self.assertEqual(A		,		{"""sequence""": ANY(A    ), """labels""": [ANY(A    )], """scores""": [ANY(A    )]}    )

								# No kwarg
								UpperCAmelCase     :		Optional[int]							     =		classifier("""Who are you voting for in 2020?"""		,		["""politics"""]    )
								self.assertEqual(A		,		{"""sequence""": ANY(A    ), """labels""": [ANY(A    )], """scores""": [ANY(A    )]}    )

								UpperCAmelCase     :		str							     =		classifier("""Who are you voting for in 2020?"""		,		candidate_labels=["""politics"""]    )
								self.assertEqual(A		,		{"""sequence""": ANY(A    ), """labels""": [ANY(A    )], """scores""": [ANY(A    )]}    )

								UpperCAmelCase     :		List[Any]							     =		classifier("""Who are you voting for in 2020?"""		,		candidate_labels="""politics, public health"""    )
								self.assertEqual(
								    A		,		{"""sequence""": ANY(A    ), """labels""": [ANY(A    ), ANY(A    )], """scores""": [ANY(A    ), ANY(A    )]}    )
								self.assertAlmostEqual(sum(nested_simplify(outputs["""scores"""]    )    )		,		1.0    )

								UpperCAmelCase     :		Optional[int]							     =		classifier("""Who are you voting for in 2020?"""		,		candidate_labels=["""politics""", """public health"""]    )
								self.assertEqual(
								    A		,		{"""sequence""": ANY(A    ), """labels""": [ANY(A    ), ANY(A    )], """scores""": [ANY(A    ), ANY(A    )]}    )
								self.assertAlmostEqual(sum(nested_simplify(outputs["""scores"""]    )    )		,		1.0    )

								UpperCAmelCase     :		Dict							     =		classifier(
								    """Who are you voting for in 2020?"""		,		candidate_labels="""politics"""		,		hypothesis_template="""This text is about {}"""    )
								self.assertEqual(A		,		{"""sequence""": ANY(A    ), """labels""": [ANY(A    )], """scores""": [ANY(A    )]}    )

								# https://github.com/huggingface/transformers/issues/13846
								UpperCAmelCase     :		str							     =		classifier(["""I am happy"""]		,		["""positive""", """negative"""]    )
								self.assertEqual(
								    A		,		[
								        {"""sequence""": ANY(A    ), """labels""": [ANY(A    ), ANY(A    )], """scores""": [ANY(A    ), ANY(A    )]}
								        for i in range(1    )
								    ]		,		)
								UpperCAmelCase     :		List[str]							     =		classifier(["""I am happy""", """I am sad"""]		,		["""positive""", """negative"""]    )
								self.assertEqual(
								    A		,		[
								        {"""sequence""": ANY(A    ), """labels""": [ANY(A    ), ANY(A    )], """scores""": [ANY(A    ), ANY(A    )]}
								        for i in range(2    )
								    ]		,		)

								with self.assertRaises(A    ):
															classifier(""""""		,		candidate_labels="""politics"""    )

								with self.assertRaises(A    ):
															classifier(A		,		candidate_labels="""politics"""    )

								with self.assertRaises(A    ):
															classifier("""Who are you voting for in 2020?"""		,		candidate_labels=""""""    )

								with self.assertRaises(A    ):
															classifier("""Who are you voting for in 2020?"""		,		candidate_labels=A    )

								with self.assertRaises(A    ):
															classifier(
															    """Who are you voting for in 2020?"""		,		candidate_labels="""politics"""		,		hypothesis_template="""Not formatting template"""		,		)

								with self.assertRaises(A    ):
															classifier(
															    """Who are you voting for in 2020?"""		,		candidate_labels="""politics"""		,		hypothesis_template=A		,		)

								self.run_entailment_id(A    )





	def  _lowercase(  self		,		A    )			->			Any:
								UpperCAmelCase     :		Tuple							     =		zero_shot_classifier.model.config
								UpperCAmelCase     :		Union[str, Any]							     =		config.labelaid
								UpperCAmelCase     :		Tuple							     =		zero_shot_classifier.entailment_id

								UpperCAmelCase     :		Any							     =		{"""LABEL_0""": 0, """LABEL_1""": 1, """LABEL_2""": 2}
								self.assertEqual(zero_shot_classifier.entailment_id		,		-1    )

								UpperCAmelCase     :		Optional[Any]							     =		{"""entailment""": 0, """neutral""": 1, """contradiction""": 2}
								self.assertEqual(zero_shot_classifier.entailment_id		,		0    )

								UpperCAmelCase     :		Any							     =		{"""ENTAIL""": 0, """NON-ENTAIL""": 1}
								self.assertEqual(zero_shot_classifier.entailment_id		,		0    )

								UpperCAmelCase     :		List[str]							     =		{"""ENTAIL""": 2, """NEUTRAL""": 1, """CONTR""": 0}
								self.assertEqual(zero_shot_classifier.entailment_id		,		2    )

								UpperCAmelCase     :		Tuple							     =		original_labelaid
								self.assertEqual(A		,		zero_shot_classifier.entailment_id    )





	@require_torch
	def  _lowercase(  self    )			->			str:
								UpperCAmelCase     :		int							     =		pipeline(
								    """zero-shot-classification"""		,		model="""sshleifer/tiny-distilbert-base-cased-distilled-squad"""		,		framework="""pt"""		,		)
								# There was a regression in 4.10 for this
								# Adding a test so we don't make the mistake again.
								# https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
								zero_shot_classifier(
								    """Who are you voting for in 2020?""" * 100		,		candidate_labels=["""politics""", """public health""", """science"""]    )





	@require_torch
	def  _lowercase(  self    )			->			Union[str, Any]:
								UpperCAmelCase     :		Optional[int]							     =		pipeline(
								    """zero-shot-classification"""		,		model="""sshleifer/tiny-distilbert-base-cased-distilled-squad"""		,		framework="""pt"""		,		)
								UpperCAmelCase     :		Union[str, Any]							     =		zero_shot_classifier(
								    """Who are you voting for in 2020?"""		,		candidate_labels=["""politics""", """public health""", """science"""]    )

								self.assertEqual(
								    nested_simplify(A    )		,		{
								        """sequence""": """Who are you voting for in 2020?""",
								        """labels""": ["""science""", """public health""", """politics"""],
								        """scores""": [0.3_3_3, 0.3_3_3, 0.3_3_3],
								    }		,		)





	@require_tf
	def  _lowercase(  self    )			->			Optional[int]:
								UpperCAmelCase     :		Optional[Any]							     =		pipeline(
								    """zero-shot-classification"""		,		model="""sshleifer/tiny-distilbert-base-cased-distilled-squad"""		,		framework="""tf"""		,		)
								UpperCAmelCase     :		List[Any]							     =		zero_shot_classifier(
								    """Who are you voting for in 2020?"""		,		candidate_labels=["""politics""", """public health""", """science"""]    )

								self.assertEqual(
								    nested_simplify(A    )		,		{
								        """sequence""": """Who are you voting for in 2020?""",
								        """labels""": ["""science""", """public health""", """politics"""],
								        """scores""": [0.3_3_3, 0.3_3_3, 0.3_3_3],
								    }		,		)





	@slow
	@require_torch
	def  _lowercase(  self    )			->			List[str]:
								UpperCAmelCase     :		Optional[int]							     =		pipeline("""zero-shot-classification"""		,		model="""roberta-large-mnli"""		,		framework="""pt"""    )
								UpperCAmelCase     :		Optional[int]							     =		zero_shot_classifier(
								    """Who are you voting for in 2020?"""		,		candidate_labels=["""politics""", """public health""", """science"""]    )

								self.assertEqual(
								    nested_simplify(A    )		,		{
								        """sequence""": """Who are you voting for in 2020?""",
								        """labels""": ["""politics""", """public health""", """science"""],
								        """scores""": [0.9_7_6, 0.0_1_5, 0.0_0_9],
								    }		,		)
								UpperCAmelCase     :		str							     =		zero_shot_classifier(
								    """The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"""
								    """ in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"""
								    """ through an attention mechanism. We propose a new simple network architecture, the Transformer, based"""
								    """ solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"""
								    """ machine translation tasks show these models to be superior in quality while being more parallelizable"""
								    """ and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"""
								    """ English-to-German translation task, improving over the existing best results, including ensembles by"""
								    """ over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"""
								    """ single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"""
								    """ fraction of the training costs of the best models from the literature. We show that the Transformer"""
								    """ generalizes well to other tasks by applying it successfully to English constituency parsing both with"""
								    """ large and limited training data."""		,		candidate_labels=["""machine learning""", """statistics""", """translation""", """vision"""]		,		multi_label=A		,		)
								self.assertEqual(
								    nested_simplify(A    )		,		{
								        """sequence""": (
								            """The dominant sequence transduction models are based on complex recurrent or convolutional neural"""
								            """ networks in an encoder-decoder configuration. The best performing models also connect the"""
								            """ encoder and decoder through an attention mechanism. We propose a new simple network"""
								            """ architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"""
								            """ and convolutions entirely. Experiments on two machine translation tasks show these models to be"""
								            """ superior in quality while being more parallelizable and requiring significantly less time to"""
								            """ train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"""
								            """ improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"""
								            """ English-to-French translation task, our model establishes a new single-model state-of-the-art"""
								            """ BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"""
								            """ costs of the best models from the literature. We show that the Transformer generalizes well to"""
								            """ other tasks by applying it successfully to English constituency parsing both with large and"""
								            """ limited training data."""
								        ),
								        """labels""": ["""translation""", """machine learning""", """vision""", """statistics"""],
								        """scores""": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
								    }		,		)






	@slow
	@require_tf
	def  _lowercase(  self    )			->			List[str]:
								UpperCAmelCase     :		int							     =		pipeline("""zero-shot-classification"""		,		model="""roberta-large-mnli"""		,		framework="""tf"""    )
								UpperCAmelCase     :		Tuple							     =		zero_shot_classifier(
								    """Who are you voting for in 2020?"""		,		candidate_labels=["""politics""", """public health""", """science"""]    )

								self.assertEqual(
								    nested_simplify(A    )		,		{
								        """sequence""": """Who are you voting for in 2020?""",
								        """labels""": ["""politics""", """public health""", """science"""],
								        """scores""": [0.9_7_6, 0.0_1_5, 0.0_0_9],
								    }		,		)
								UpperCAmelCase     :		Any							     =		zero_shot_classifier(
								    """The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"""
								    """ in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"""
								    """ through an attention mechanism. We propose a new simple network architecture, the Transformer, based"""
								    """ solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"""
								    """ machine translation tasks show these models to be superior in quality while being more parallelizable"""
								    """ and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"""
								    """ English-to-German translation task, improving over the existing best results, including ensembles by"""
								    """ over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"""
								    """ single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"""
								    """ fraction of the training costs of the best models from the literature. We show that the Transformer"""
								    """ generalizes well to other tasks by applying it successfully to English constituency parsing both with"""
								    """ large and limited training data."""		,		candidate_labels=["""machine learning""", """statistics""", """translation""", """vision"""]		,		multi_label=A		,		)
								self.assertEqual(
								    nested_simplify(A    )		,		{
								        """sequence""": (
								            """The dominant sequence transduction models are based on complex recurrent or convolutional neural"""
								            """ networks in an encoder-decoder configuration. The best performing models also connect the"""
								            """ encoder and decoder through an attention mechanism. We propose a new simple network"""
								            """ architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"""
								            """ and convolutions entirely. Experiments on two machine translation tasks show these models to be"""
								            """ superior in quality while being more parallelizable and requiring significantly less time to"""
								            """ train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"""
								            """ improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"""
								            """ English-to-French translation task, our model establishes a new single-model state-of-the-art"""
								            """ BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"""
								            """ costs of the best models from the literature. We show that the Transformer generalizes well to"""
								            """ other tasks by applying it successfully to English constituency parsing both with large and"""
								            """ limited training data."""
								        ),
								        """labels""": ["""translation""", """machine learning""", """vision""", """statistics"""],
								        """scores""": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
								    }		,		)

