
"""simple docstring"""




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 			snake_case  (				__lowercase							,      unittest.TestCase ):
							UpperCAmelCase__      =   GPTSanJapaneseTokenizer
							UpperCAmelCase__      =   False
							UpperCAmelCase__      =   {'''do_clean_text''': False, '''add_prefix_space''': False}

							def _lowercase       (self     ):

									"""simple docstring"""







									super().setUp()

									# fmt: off
									SCREAMING_SNAKE_CASE_    			=			['''こん''', '''こんに''', '''にちは''', '''ばんは''', '''世界,㔺界''', '''、''', '''。''', '''<BR>''', '''<SP>''', '''<TAB>''', '''<URL>''', '''<EMAIL>''', '''<TEL>''', '''<DATE>''', '''<PRICE>''', '''<BLOCK>''', '''<KIGOU>''', '''<U2000U2BFF>''', '''<|emoji1|>''', '''<unk>''', '''<|bagoftoken|>''', '''<|endoftext|>''']
									# fmt: on
									SCREAMING_SNAKE_CASE_    			=			{'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
									SCREAMING_SNAKE_CASE_    			=			{'''unk_token''': '''<unk>'''}

									SCREAMING_SNAKE_CASE_    			=			os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['''vocab_file''']     )
									SCREAMING_SNAKE_CASE_    			=			os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['''emoji_file''']     )
									with open(self.vocab_file							,							'''w'''							,							encoding='''utf-8'''     ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]     )     )
									with open(self.emoji_file							,							'''w'''     ) as emoji_writer:
											emoji_writer.write(json.dumps(SCREAMING_SNAKE_CASE_     )     )

							def _lowercase       (self							,							**SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									kwargs.update(self.special_tokens_map     )
									return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,							**SCREAMING_SNAKE_CASE_     )

							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、世界。 \nこんばんは、㔺界。😀'''
									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、世界。 \nこんばんは、世界。😀'''
									return input_text, output_text

							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_    			=			self.get_input_output_texts(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode(SCREAMING_SNAKE_CASE_							,							add_special_tokens=SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.decode(SCREAMING_SNAKE_CASE_							,							clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_     )
									return text, ids

							def _lowercase       (self     ):

									"""simple docstring"""







									pass  # TODO add if relevant

							def _lowercase       (self     ):

									"""simple docstring"""







									pass  # TODO add if relevant

							def _lowercase       (self     ):

									"""simple docstring"""







									pass  # TODO add if relevant

							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.get_tokenizer()

									# Testing tokenization
									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、世界。　こんばんは、㔺界。'''
									SCREAMING_SNAKE_CASE_    			=			['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
									SCREAMING_SNAKE_CASE_    			=			tokenizer.tokenize(SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

									# Testing conversion to ids without special tokens
									SCREAMING_SNAKE_CASE_    			=			[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
									SCREAMING_SNAKE_CASE_    			=			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

									# Testing conversion to ids with special tokens
									SCREAMING_SNAKE_CASE_    			=			tokens + [tokenizer.unk_token]
									SCREAMING_SNAKE_CASE_    			=			[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
									SCREAMING_SNAKE_CASE_    			=			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.get_tokenizer()

									# Testing tokenization
									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'''
									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、、、、世界。こんばんは、、、、世界。'''
									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.decode(SCREAMING_SNAKE_CASE_     )
									self.assertEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

							@slow
							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''     )

									# Testing tokenization
									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、世界。'''
									SCREAMING_SNAKE_CASE_    			=			'''こんばんは、㔺界。😀'''
									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、世界。こんばんは、世界。😀'''
									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode(prefix_text + input_text     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode(''''''							,							prefix_text=prefix_text + input_text     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode(SCREAMING_SNAKE_CASE_							,							prefix_text=SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.decode(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.decode(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.decode(SCREAMING_SNAKE_CASE_     )
									self.assertEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									self.assertEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									self.assertEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

							@slow
							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''     )

									# Testing tokenization
									SCREAMING_SNAKE_CASE_    			=			'''こんにちは、世界。'''
									SCREAMING_SNAKE_CASE_    			=			'''こんばんは、㔺界。😀'''

									SCREAMING_SNAKE_CASE_    			=			len(tokenizer.encode(SCREAMING_SNAKE_CASE_     )     ) - 2
									SCREAMING_SNAKE_CASE_    			=			len(tokenizer.encode(SCREAMING_SNAKE_CASE_     )     ) - 2

									SCREAMING_SNAKE_CASE_    			=			[1] + [0] * (len_prefix + len_text + 1)
									SCREAMING_SNAKE_CASE_    			=			[1] * (len_prefix + len_text + 1) + [0]
									SCREAMING_SNAKE_CASE_    			=			[1] + [1] * (len_prefix) + [0] * (len_text + 1)

									SCREAMING_SNAKE_CASE_    			=			tokenizer(prefix_text + input_text     ).token_type_ids
									SCREAMING_SNAKE_CASE_    			=			tokenizer(''''''							,							prefix_text=prefix_text + input_text     ).token_type_ids
									SCREAMING_SNAKE_CASE_    			=			tokenizer(SCREAMING_SNAKE_CASE_							,							prefix_text=SCREAMING_SNAKE_CASE_     ).token_type_ids
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

							@slow
							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''     )

									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode('''あンいワ'''     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode(''''''							,							prefix_text='''あンいワ'''     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.encode('''いワ'''							,							prefix_text='''あン'''     )

									self.assertEqual(tokenizer.decode(SCREAMING_SNAKE_CASE_     )							,							tokenizer.decode(SCREAMING_SNAKE_CASE_     )     )
									self.assertEqual(tokenizer.decode(SCREAMING_SNAKE_CASE_     )							,							tokenizer.decode(SCREAMING_SNAKE_CASE_     )     )
									self.assertNotEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									self.assertNotEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									self.assertEqual(x_token_a[1]							,							x_token_a[-1]     )  # SEG token
									self.assertEqual(x_token_a[1]							,							x_token_a[3]     )  # SEG token

							@slow
							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''     )

									SCREAMING_SNAKE_CASE_    			=			[['''武田信玄''', '''は、'''], ['''織田信長''', '''の配下の、''']]
									SCREAMING_SNAKE_CASE_    			=			tokenizer(SCREAMING_SNAKE_CASE_							,							padding=SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			tokenizer.batch_encode_plus(SCREAMING_SNAKE_CASE_							,							padding=SCREAMING_SNAKE_CASE_     )

									# fmt: off
									SCREAMING_SNAKE_CASE_    			=			[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
									SCREAMING_SNAKE_CASE_    			=			[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
									SCREAMING_SNAKE_CASE_    			=			[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
									# fmt: on
									self.assertListEqual(x_token.input_ids							,							SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(x_token.token_type_ids							,							SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(x_token.attention_mask							,							SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(x_token_a.input_ids							,							SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(x_token_a.token_type_ids							,							SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(x_token_a.attention_mask							,							SCREAMING_SNAKE_CASE_     )

							def _lowercase       (self     ):

									"""simple docstring"""







									pass







							def _lowercase       (self     ):

									"""simple docstring"""







									pass

"""simple docstring"""




import datasets

from .evaluate import evaluate


lowerCAmelCase__				=					'\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n'

lowerCAmelCase__				=					'\nThis metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n'

lowerCAmelCase__				=					'\nComputes SQuAD scores (F1 and EM).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair as given in the references (see below)\n        - \'prediction_text\': the text of the answer\n    references: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair (see above),\n        - \'answers\': a Dict in the SQuAD dataset format\n            {\n                \'text\': list of possible texts for the answer, as a list of strings\n                \'answer_start\': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)\n    \'f1\': The F-score of predicted tokens versus the gold answer\nExamples:\n\n    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> squad_metric = datasets.load_metric("squad")\n    >>> results = squad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 100.0, \'f1\': 100.0}\n'





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,      _KWARGS_DESCRIPTION )
class 			snake_case  (				datasets.Metric ):

							def _lowercase       (self     ):

									"""simple docstring"""







									return datasets.MetricInfo(
									    description=_DESCRIPTION							,							citation=_CITATION							,							inputs_description=_KWARGS_DESCRIPTION							,							features=datasets.Features(
									        {
									            '''predictions''': {'''id''': datasets.Value('''string'''     ), '''prediction_text''': datasets.Value('''string'''     )},
									            '''references''': {
									                '''id''': datasets.Value('''string'''     ),
									                '''answers''': datasets.features.Sequence(
									                    {
									                        '''text''': datasets.Value('''string'''     ),
									                        '''answer_start''': datasets.Value('''int32'''     ),
									                    }     ),
									            },
									        }     )							,							codebase_urls=['''https://rajpurkar.github.io/SQuAD-explorer/''']							,							reference_urls=['''https://rajpurkar.github.io/SQuAD-explorer/''']							,							)







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			{prediction['''id''']: prediction['''prediction_text'''] for prediction in predictions}
									SCREAMING_SNAKE_CASE_    			=			[
									    {
									        '''paragraphs''': [
									            {
									                '''qas''': [
									                    {
									                        '''answers''': [{'''text''': answer_text} for answer_text in ref['''answers''']['''text''']],
									                        '''id''': ref['''id'''],
									                    }
									                    for ref in references
									                ]
									            }
									        ]
									    }
									]
									SCREAMING_SNAKE_CASE_    			=			evaluate(dataset=SCREAMING_SNAKE_CASE_							,							predictions=SCREAMING_SNAKE_CASE_     )
									return score