




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class    lowercase_		(SCREAMING_SNAKE_CASE__							,	unittest.TestCase ):
						lowerCAmelCase__			      =BertJapaneseTokenizer
						lowerCAmelCase__			      =False
						lowerCAmelCase__			      =True



						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								super().setUp()

								SCREAMING_SNAKE_CASE_					      =							[
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "こんにちは",
								    "こん",
								    "にちは",
								    "ばんは",
								    "##こん",
								    "##にちは",
								    "##ばんは",
								    "世界",
								    "##世界",
								    "、",
								    "##、",
								    "。",
								    "##。",
								]

								SCREAMING_SNAKE_CASE_					      =							os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES['vocab_file']  )
								with open(self.vocab_file					,							'w'					,							encoding='utf-8'  ) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )



						def 	__a					(			self	:   List[str]					,							snake_case__	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							"こんにちは、世界。 \nこんばんは、世界。"
								SCREAMING_SNAKE_CASE_					      =							"こんにちは 、 世界 。 こんばんは 、 世界 。"
								return input_text, output_text



						def 	__a					(			self	:   Optional[Any]					,							snake_case__	:   Any  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.get_input_output_texts(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode(snake_case__					,							add_special_tokens=snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.decode(snake_case__					,							clean_up_tokenization_spaces=snake_case__  )
								return text, ids



						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   Dict  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   Dict  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class(self.vocab_file  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'  )
								self.assertListEqual(snake_case__					,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__  )					,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )



						def 	__a					(			self	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class(self.vocab_file					,							word_tokenizer_type='mecab'  )
								self.assertIsNotNone(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							"こんにちは、世界。\nこんばんは、世界。"
								SCREAMING_SNAKE_CASE_					      =							tokenizer.tokenize(snake_case__  )
								self.assertListEqual(snake_case__					,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__  )					,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

								SCREAMING_SNAKE_CASE_					      =							os.path.join(self.tmpdirname					,							'tokenizer.bin'  )
								with open(snake_case__					,							'wb'  ) as handle:
										pickle.dump(snake_case__					,							snake_case__  )

								with open(snake_case__					,							'rb'  ) as handle:
										SCREAMING_SNAKE_CASE_					      =							pickle.load(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer_new.tokenize(snake_case__  )

								self.assertListEqual(snake_case__					,							snake_case__  )



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							MecabTokenizer(mecab_dic='ipadic'  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,							)



						def 	__a					(			self	:   Any  ):

								"""simple docstring"""

								try:
										SCREAMING_SNAKE_CASE_					      =							MecabTokenizer(mecab_dic='unidic_lite'  )
								except ModuleNotFoundError:
										return

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,							)



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								try:
										SCREAMING_SNAKE_CASE_					      =							MecabTokenizer(mecab_dic='unidic'  )
								except ModuleNotFoundError:
										return

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,							)



						def 	__a					(			self	:   str  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							MecabTokenizer(do_lower_case=snake_case__					,							mecab_dic='ipadic'  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,							)



						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								try:
										SCREAMING_SNAKE_CASE_					      =							MecabTokenizer(
										    do_lower_case=snake_case__					,							normalize_text=snake_case__					,							mecab_option='-d /usr/local/lib/mecab/dic/jumandic'  )
								except RuntimeError:
										# if dict doesn't exist in the system, previous code raises this error.
										return

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,							)



						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							MecabTokenizer(normalize_text=snake_case__					,							mecab_dic='ipadic'  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,							)



						@require_sudachi
						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class(self.vocab_file					,							word_tokenizer_type='sudachi'  )
								self.assertIsNotNone(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							"こんにちは、世界。\nこんばんは、世界。"
								SCREAMING_SNAKE_CASE_					      =							tokenizer.tokenize(snake_case__  )
								self.assertListEqual(snake_case__					,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__  )					,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

								SCREAMING_SNAKE_CASE_					      =							os.path.join(self.tmpdirname					,							'tokenizer.bin'  )
								with open(snake_case__					,							'wb'  ) as handle:
										pickle.dump(snake_case__					,							snake_case__  )

								with open(snake_case__					,							'rb'  ) as handle:
										SCREAMING_SNAKE_CASE_					      =							pickle.load(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer_new.tokenize(snake_case__  )

								self.assertListEqual(snake_case__					,							snake_case__  )



						@require_sudachi
						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							SudachiTokenizer(sudachi_dict_type='core'  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							[' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,							)



						@require_sudachi
						def 	__a					(			self	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							SudachiTokenizer(sudachi_dict_type='core'					,							sudachi_split_mode='A'  )

								self.assertListEqual(tokenizer.tokenize('外国人参政権'  )					,							['外国', '人', '参政', '権']  )



						@require_sudachi
						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							SudachiTokenizer(sudachi_dict_type='core'					,							sudachi_split_mode='B'  )

								self.assertListEqual(tokenizer.tokenize('外国人参政権'  )					,							['外国人', '参政権']  )



						@require_sudachi
						def 	__a					(			self	:   Dict  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							SudachiTokenizer(sudachi_dict_type='core'					,							sudachi_split_mode='C'  )

								self.assertListEqual(tokenizer.tokenize('外国人参政権'  )					,							['外国人参政権']  )



						@require_sudachi
						def 	__a					(			self	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							SudachiTokenizer(do_lower_case=snake_case__					,							sudachi_dict_type='core'  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							[' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,							)



						@require_sudachi
						def 	__a					(			self	:   Tuple  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							SudachiTokenizer(normalize_text=snake_case__					,							sudachi_dict_type='core'  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							[' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,							)



						@require_sudachi
						def 	__a					(			self	:   Dict  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							SudachiTokenizer(trim_whitespace=snake_case__					,							sudachi_dict_type='core'  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,							)



						@require_jumanpp
						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class(self.vocab_file					,							word_tokenizer_type='jumanpp'  )
								self.assertIsNotNone(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							"こんにちは、世界。\nこんばんは、世界。"
								SCREAMING_SNAKE_CASE_					      =							tokenizer.tokenize(snake_case__  )
								self.assertListEqual(snake_case__					,							['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__  )					,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

								SCREAMING_SNAKE_CASE_					      =							os.path.join(self.tmpdirname					,							'tokenizer.bin'  )
								with open(snake_case__					,							'wb'  ) as handle:
										pickle.dump(snake_case__					,							snake_case__  )

								with open(snake_case__					,							'rb'  ) as handle:
										SCREAMING_SNAKE_CASE_					      =							pickle.load(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer_new.tokenize(snake_case__  )

								self.assertListEqual(snake_case__					,							snake_case__  )



						@require_jumanpp
						def 	__a					(			self	:   str  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,							)



						@require_jumanpp
						def 	__a					(			self	:   Dict  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							JumanppTokenizer(do_lower_case=snake_case__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,							)



						@require_jumanpp
						def 	__a					(			self	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							JumanppTokenizer(normalize_text=snake_case__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,							)



						@require_jumanpp
						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							JumanppTokenizer(trim_whitespace=snake_case__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )					,							['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,							)



						@require_jumanpp
						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'  )					,							['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,							)



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

								SCREAMING_SNAKE_CASE_					      =							{}
								for i, token in enumerate(snake_case__  ):
										SCREAMING_SNAKE_CASE_					      =							i
								SCREAMING_SNAKE_CASE_					      =							WordpieceTokenizer(vocab=snake_case__					,							unk_token='[UNK]'  )

								self.assertListEqual(tokenizer.tokenize(''  )					,							[]  )

								self.assertListEqual(tokenizer.tokenize('こんにちは'  )					,							['こんにちは']  )

								self.assertListEqual(tokenizer.tokenize('こんばんは'  )					,							['こん', '##ばんは']  )

								self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'  )					,							['こん', '##ばんは', '[UNK]', 'こんにちは']  )



						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.subword_tokenizer

								SCREAMING_SNAKE_CASE_					      =							subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'  )
								self.assertListEqual(snake_case__					,							['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']  )

								SCREAMING_SNAKE_CASE_					      =							subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'  )
								self.assertListEqual(snake_case__					,							['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']  )






						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode('ありがとう。'					,							add_special_tokens=snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode('どういたしまして。'					,							add_special_tokens=snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer.build_inputs_with_special_tokens(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.build_inputs_with_special_tokens(snake_case__					,							snake_case__  )

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class    lowercase_		(SCREAMING_SNAKE_CASE__							,	unittest.TestCase ):
						lowerCAmelCase__			      =BertJapaneseTokenizer
						lowerCAmelCase__			      =False



						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								super().setUp()

								SCREAMING_SNAKE_CASE_					      =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

								SCREAMING_SNAKE_CASE_					      =							os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES['vocab_file']  )
								with open(self.vocab_file					,							'w'					,							encoding='utf-8'  ) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )



						def 	__a					(			self	:   int					,							**snake_case__	:   Any  ):

								"""simple docstring"""

								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,							subword_tokenizer_type='character'					,							**snake_case__  )



						def 	__a					(			self	:   Union[str, Any]					,							snake_case__	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							"こんにちは、世界。 \nこんばんは、世界。"
								SCREAMING_SNAKE_CASE_					      =							"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
								return input_text, output_text



						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class(self.vocab_file					,							subword_tokenizer_type='character'  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'  )
								self.assertListEqual(
								    snake_case__					,							['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']  )
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(snake_case__  )					,							[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]  )



						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

								SCREAMING_SNAKE_CASE_					      =							{}
								for i, token in enumerate(snake_case__  ):
										SCREAMING_SNAKE_CASE_					      =							i
								SCREAMING_SNAKE_CASE_					      =							CharacterTokenizer(vocab=snake_case__					,							unk_token='[UNK]'  )

								self.assertListEqual(tokenizer.tokenize(''  )					,							[]  )

								self.assertListEqual(tokenizer.tokenize('こんにちは'  )					,							['こ', 'ん', 'に', 'ち', 'は']  )

								self.assertListEqual(tokenizer.tokenize('こんにちほ'  )					,							['こ', 'ん', 'に', 'ち', '[UNK]']  )






						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode('ありがとう。'					,							add_special_tokens=snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode('どういたしまして。'					,							add_special_tokens=snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer.build_inputs_with_special_tokens(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.build_inputs_with_special_tokens(snake_case__					,							snake_case__  )

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class    lowercase_		(unittest.TestCase ):



						def 	__a					(			self	:   Any  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							"cl-tohoku/bert-base-japanese"
								SCREAMING_SNAKE_CASE_					      =							AutoTokenizer.from_pretrained(snake_case__  )
								self.assertIsInstance(snake_case__					,							snake_case__  )



class    lowercase_		(unittest.TestCase ):



						def 	__a					(			self	:   Tuple  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							"cl-tohoku/bert-base-japanese"
								with self.assertLogs('transformers'					,							level='WARNING'  ) as cm:
										BertTokenizer.from_pretrained(snake_case__  )
										self.assertTrue(
										    cm.records[0].message.startswith(
										        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
										        ' is called from.'  )  )
								SCREAMING_SNAKE_CASE_					      =							"bert-base-cased"
								with self.assertLogs('transformers'					,							level='WARNING'  ) as cm:
										BertJapaneseTokenizer.from_pretrained(snake_case__  )
										self.assertTrue(
										    cm.records[0].message.startswith(
										        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
										        ' is called from.'  )  )



import os

from bleurt import score  # From: git+https://github.com/google-research/bleurt.git

import datasets


SCREAMING_SNAKE_CASE__     					=			datasets.logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__     					=			"\\n@inproceedings{bleurt,\n  title={BLEURT: Learning Robust Metrics for Text Generation},\n  author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh},\n  booktitle={ACL},\n  year={2020},\n  url={https://arxiv.org/abs/2004.04696}\n}\n"

SCREAMING_SNAKE_CASE__     					=			"\\nBLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018)\nand then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune\nit for your specific application (the latter is expected to perform better).\n\nSee the project's README at https://github.com/google-research/bleurt#readme for more information.\n"

SCREAMING_SNAKE_CASE__     					=			"\nBLEURT score.\n\nArgs:\n    `predictions` (list of str): prediction/candidate sentences\n    `references` (list of str): reference sentences\n    `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None.\n\nReturns:\n    'scores': List of scores.\nExamples:\n\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> bleurt = datasets.load_metric(\"bleurt\")\n    >>> results = bleurt.compute(predictions=predictions, references=references)\n    >>> print([round(v, 2) for v in results[\"scores\"]])\n    [1.03, 1.04]\n"

SCREAMING_SNAKE_CASE__     					=			{
    "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip",
    "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip",
    "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip",
    "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip",
    "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip",
    "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip",
    "BLEURT-20-D3": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip",
    "BLEURT-20-D6": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip",
    "BLEURT-20-D12": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip",
    "BLEURT-20": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip",
}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,		_KWARGS_DESCRIPTION							)
class 						_UpperCAmelCase      (     datasets.Metric							):






		def _snake_case		(						self				:     List[str]):
				return datasets.MetricInfo(
				    description=_DESCRIPTION   ,  citation=_CITATION   ,  homepage="https://github.com/google-research/bleurt"   ,  inputs_description=_KWARGS_DESCRIPTION   ,  features=datasets.Features(
				        {
				            "predictions": datasets.Value("string"   ,  id="sequence"),
				            "references": datasets.Value("string"   ,  id="sequence"),
				        })   ,  codebase_urls=["https://github.com/google-research/bleurt"]   ,  reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"]   ,  )






		def _snake_case		(						self				:     Any   ,  UpperCAmelCase				:     Any):
				# check that config name specifies a valid BLEURT model
				if self.config_name == "default":
						logger.warning(
						    "Using default BLEURT-Base checkpoint for sequence maximum length 128. "
						    "You can use a bigger model for better results with e.g.: datasets.load_metric('bleurt', 'bleurt-large-512').")
						SCREAMING_SNAKE_CASE_				:str				     =						"bleurt-base-128"

				if self.config_name.lower() in CHECKPOINT_URLS:
						SCREAMING_SNAKE_CASE_				:Dict				     =						self.config_name.lower()

				elif self.config_name.upper() in CHECKPOINT_URLS:
						SCREAMING_SNAKE_CASE_				:Dict				     =						self.config_name.upper()

				else:
						raise KeyError(
						    F"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}")

				# download the model checkpoint specified by self.config_name and set up the scorer
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name])
				SCREAMING_SNAKE_CASE_				:Dict				     =						score.BleurtScorer(os.path.join(UpperCAmelCase   ,  UpperCAmelCase))







		def _snake_case		(						self				:     Union[str, Any]   ,  UpperCAmelCase				:     List[str]   ,  UpperCAmelCase				:     Union[str, Any]):
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						self.scorer.score(references=UpperCAmelCase   ,  candidates=UpperCAmelCase)
				return {"scores": scores}






