





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer



class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		int     )							->						Union[str, Any]:
     __lowerCAmelCase								=	tempfile.mkdtemp()
     __lowerCAmelCase								=	5

     # Realm tok
     __lowerCAmelCase								=	[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         '[PAD]',
         '[MASK]',
         'test',
         'question',
         'this',
         'is',
         'the',
         'first',
         'second',
         'third',
         'fourth',
         'fifth',
         'record',
         'want',
         '##want',
         '##ed',
         'wa',
         'un',
         'runn',
         '##ing',
         ',',
         'low',
         'lowest',
     ]
     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'realm_tokenizer'     )
     os.makedirs(lowerCAmelCase_					,  exist_ok=lowerCAmelCase_     )
     __lowerCAmelCase								=	os.path.join(lowerCAmelCase_					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'realm_block_records'     )
     os.makedirs(lowerCAmelCase_					,  exist_ok=lowerCAmelCase_     )






  def 				lowercase		(				self :		Tuple     )							->						RealmTokenizer:
     return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname					,  'realm_tokenizer'     )     )






  def 				lowercase		(				self :		int     )							->						Optional[Any]:
     shutil.rmtree(self.tmpdirname     )






  def 				lowercase		(				self :		Optional[Any]     )							->						List[Any]:
     __lowerCAmelCase								=	RealmConfig(num_block_records=self.num_block_records     )
     return config






  def 				lowercase		(				self :		int     )							->						List[Any]:
     __lowerCAmelCase								=	Dataset.from_dict(
         {
             'id': ['0', '1'],
             'question': ['foo', 'bar'],
             'answers': [['Foo', 'Bar'], ['Bar']],
         }     )
     return dataset






  def 				lowercase		(				self :		Optional[int]     )							->						Optional[int]:
     __lowerCAmelCase								=	np.array(
         [
             B'This is the first record',
             B'This is the second record',
             B'This is the third record',
             B'This is the fourth record',
             B'This is the fifth record',
             B'This is a longer longer longer record',
         ]					,  dtype=lowerCAmelCase_					,  )
     return block_records






  def 				lowercase		(				self :		str     )							->						int:
     __lowerCAmelCase								=	RealmRetriever(
         block_records=self.get_dummy_block_records()					,  tokenizer=self.get_tokenizer()					,  )
     return retriever






  def 				lowercase		(				self :		Union[str, Any]     )							->						List[Any]:
     __lowerCAmelCase								=	self.get_config()
     __lowerCAmelCase								=	self.get_dummy_retriever()
     __lowerCAmelCase								=	retriever.tokenizer

     __lowerCAmelCase								=	np.array([0, 3]					,  dtype='long'     )
     __lowerCAmelCase								=	tokenizer(['Test question']     ).input_ids
     __lowerCAmelCase								=	tokenizer(
         ['the fourth']					,  add_special_tokens=lowerCAmelCase_					,  return_token_type_ids=lowerCAmelCase_					,  return_attention_mask=lowerCAmelCase_					,  ).input_ids
     __lowerCAmelCase								=	config.reader_seq_len

     __lowerCAmelCase	,	__lowerCAmelCase	,	__lowerCAmelCase	,	__lowerCAmelCase								=	retriever(
         lowerCAmelCase_					,  lowerCAmelCase_					,  answer_ids=lowerCAmelCase_					,  max_length=lowerCAmelCase_					,  return_tensors='np'     )

     self.assertEqual(len(lowerCAmelCase_     )					,  2     )
     self.assertEqual(len(lowerCAmelCase_     )					,  2     )
     self.assertEqual(len(lowerCAmelCase_     )					,  2     )
     self.assertEqual(concat_inputs.input_ids.shape					,  (2, 1_0)     )
     self.assertEqual(concat_inputs.attention_mask.shape					,  (2, 1_0)     )
     self.assertEqual(concat_inputs.token_type_ids.shape					,  (2, 1_0)     )
     self.assertEqual(concat_inputs.special_tokens_mask.shape					,  (2, 1_0)     )
     self.assertEqual(
         tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]     )					,  ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']					,  )
     self.assertEqual(
         tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]     )					,  ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']					,  )






  def 				lowercase		(				self :		Tuple     )							->						List[Any]:
     __lowerCAmelCase								=	self.get_config()
     __lowerCAmelCase								=	self.get_dummy_retriever()
     __lowerCAmelCase								=	retriever.tokenizer

     __lowerCAmelCase								=	np.array([0, 3, 5]					,  dtype='long'     )
     __lowerCAmelCase								=	tokenizer(['Test question']     ).input_ids
     __lowerCAmelCase								=	tokenizer(
         ['the fourth', 'longer longer']					,  add_special_tokens=lowerCAmelCase_					,  return_token_type_ids=lowerCAmelCase_					,  return_attention_mask=lowerCAmelCase_					,  ).input_ids
     __lowerCAmelCase								=	config.reader_seq_len

     __lowerCAmelCase	,	__lowerCAmelCase	,	__lowerCAmelCase	,	__lowerCAmelCase								=	retriever(
         lowerCAmelCase_					,  lowerCAmelCase_					,  answer_ids=lowerCAmelCase_					,  max_length=lowerCAmelCase_					,  return_tensors='np'     )

     self.assertEqual([False, True, True]					,  lowerCAmelCase_     )
     self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]					,  lowerCAmelCase_     )
     self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		str     )							->						Dict:
     __lowerCAmelCase								=	self.get_dummy_retriever()
     retriever.save_pretrained(os.path.join(self.tmpdirname					,  'realm_block_records'     )     )

     # Test local path
     __lowerCAmelCase								=	retriever.from_pretrained(os.path.join(self.tmpdirname					,  'realm_block_records'     )     )
     self.assertEqual(retriever.block_records[0]					,  B'This is the first record'     )

     # Test mocked remote path
     with patch('transformers.models.realm.retrieval_realm.hf_hub_download'     ) as mock_hf_hub_download:
        __lowerCAmelCase								=	os.path.join(
            os.path.join(self.tmpdirname					,  'realm_block_records'     )					,  _REALM_BLOCK_RECORDS_FILENAME     )
        __lowerCAmelCase								=	RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa'     )

     self.assertEqual(retriever.block_records[0]					,  B'This is the first record'     )








import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


_snake_case  :    Tuple						    =  '\\n@article{hendrycksmath2021,\n  title={Measuring Mathematical Problem Solving With the MATH Dataset},\n  author={Dan Hendrycks\n    and Collin Burns\n    and Saurav Kadavath\n    and Akul Arora\n    and Steven Basart\n    and Eric Tang\n    and Dawn Song\n    and Jacob Steinhardt},\n  journal={arXiv preprint arXiv:2103.03874},\n  year={2021}\n}\n'


_snake_case  :    str						    =  '\\nThis metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.\nIt first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy.\n'


_snake_case  :    List[str]						    =  R'\nCalculates accuracy after canonicalizing inputs.\n\nArgs:\n    predictions: list of predictions to score. Each prediction\n        is a string that contains natural language and LaTex.\n    references: list of reference for each prediction. Each\n        reference is a string that contains natural language\n        and LaTex.\nReturns:\n    accuracy: accuracy after canonicalizing inputs\n        (e.g., converting "1/2" to "\\frac{1}{2}")\n\nExamples:\n    >>> metric = datasets.load_metric("competition_math")\n    >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"])\n    >>> print(results)\n    {\'accuracy\': 1.0}\n'



@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION		,    _KWARGS_DESCRIPTION    )
class 				_UpperCAmelCase    (    datasets.Metric    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						int:
     return datasets.MetricInfo(
         description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(
             {
                 'predictions': datasets.Value('string'     ),
                 'references': datasets.Value('string'     ),
             }     )					,  homepage='https://github.com/hendrycks/math'					,  codebase_urls=['https://github.com/hendrycks/math']					,  )






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		Any					,  lowerCAmelCase_ :		Optional[int]     )							->						List[Any]:
     __lowerCAmelCase								=	0.0
     for i, j in zip(lowerCAmelCase_					,  lowerCAmelCase_     ):
        n_correct += 1.0 if math_equivalence.is_equiv(lowerCAmelCase_					,  lowerCAmelCase_     ) else 0.0
     __lowerCAmelCase								=	n_correct / len(lowerCAmelCase_     )
     return {
         "accuracy": accuracy,
     }

