
import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer





class 							UpperCAmelCase      (  __SCREAMING_SNAKE_CASE  ):




	'''simple docstring'''





	def 				UpperCamelCase_  (       self					: List[Any] ):
					__A   			= tempfile.mkdtemp()
					__A   			= 5

					# Realm tok
					__A   			= [
					    "[UNK]",
					    "[CLS]",
					    "[SEP]",
					    "[PAD]",
					    "[MASK]",
					    "test",
					    "question",
					    "this",
					    "is",
					    "the",
					    "first",
					    "second",
					    "third",
					    "fourth",
					    "fifth",
					    "record",
					    "want",
					    "##want",
					    "##ed",
					    "wa",
					    "un",
					    "runn",
					    "##ing",
					    ",",
					    "low",
					    "lowest",
					]
					__A   			= os.path.join(self.tmpdirname							,"realm_tokenizer" )
					os.makedirs(A							,exist_ok=A )
					__A   			= os.path.join(A							,VOCAB_FILES_NAMES["vocab_file"] )
					with open(self.vocab_file							,"w"							,encoding="utf-8" ) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )

					__A   			= os.path.join(self.tmpdirname							,"realm_block_records" )
					os.makedirs(A							,exist_ok=A )






	def 				UpperCamelCase_  (       self					: int ):
					return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname							,"realm_tokenizer" ) )






	def 				UpperCamelCase_  (       self					: Optional[Any] ):
					shutil.rmtree(self.tmpdirname )






	def 				UpperCamelCase_  (       self					: int ):
					__A   			= RealmConfig(num_block_records=self.num_block_records )
					return config






	def 				UpperCamelCase_  (       self					: Union[str, Any] ):
					__A   			= Dataset.from_dict(
					    {
					        "id": ["0", "1"],
					        "question": ["foo", "bar"],
					        "answers": [["Foo", "Bar"], ["Bar"]],
					    } )
					return dataset






	def 				UpperCamelCase_  (       self					: List[Any] ):
					__A   			= np.array(
					    [
					        B"This is the first record",
					        B"This is the second record",
					        B"This is the third record",
					        B"This is the fourth record",
					        B"This is the fifth record",
					        B"This is a longer longer longer record",
					    ]							,dtype=A							,)
					return block_records






	def 				UpperCamelCase_  (       self					: Tuple ):
					__A   			= RealmRetriever(
					    block_records=self.get_dummy_block_records()							,tokenizer=self.get_tokenizer()							,)
					return retriever






	def 				UpperCamelCase_  (       self					: Optional[int] ):
					__A   			= self.get_config()
					__A   			= self.get_dummy_retriever()
					__A   			= retriever.tokenizer

					__A   			= np.array([0, 3]							,dtype="long" )
					__A   			= tokenizer(["Test question"] ).input_ids
					__A   			= tokenizer(
					    ["the fourth"]							,add_special_tokens=A							,return_token_type_ids=A							,return_attention_mask=A							,).input_ids
					__A   			= config.reader_seq_len

					__A			,							__A			,							__A			,							__A   			= retriever(
					    A							,A							,answer_ids=A							,max_length=A							,return_tensors="np" )

					self.assertEqual(len(A )							,2 )
					self.assertEqual(len(A )							,2 )
					self.assertEqual(len(A )							,2 )
					self.assertEqual(concat_inputs.input_ids.shape							,(2, 10) )
					self.assertEqual(concat_inputs.attention_mask.shape							,(2, 10) )
					self.assertEqual(concat_inputs.token_type_ids.shape							,(2, 10) )
					self.assertEqual(concat_inputs.special_tokens_mask.shape							,(2, 10) )
					self.assertEqual(
					    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0] )							,["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "first", "record", "[SEP]"]							,)
					self.assertEqual(
					    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1] )							,["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "fourth", "record", "[SEP]"]							,)






	def 				UpperCamelCase_  (       self					: Dict ):
					__A   			= self.get_config()
					__A   			= self.get_dummy_retriever()
					__A   			= retriever.tokenizer

					__A   			= np.array([0, 3, 5]							,dtype="long" )
					__A   			= tokenizer(["Test question"] ).input_ids
					__A   			= tokenizer(
					    ["the fourth", "longer longer"]							,add_special_tokens=A							,return_token_type_ids=A							,return_attention_mask=A							,).input_ids
					__A   			= config.reader_seq_len

					__A			,							__A			,							__A			,							__A   			= retriever(
					    A							,A							,answer_ids=A							,max_length=A							,return_tensors="np" )

					self.assertEqual([False, True, True]							,A )
					self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]							,A )
					self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]							,A )







	def 				UpperCamelCase_  (       self					: Tuple ):
					__A   			= self.get_dummy_retriever()
					retriever.save_pretrained(os.path.join(self.tmpdirname							,"realm_block_records" ) )

					# Test local path
					__A   			= retriever.from_pretrained(os.path.join(self.tmpdirname							,"realm_block_records" ) )
					self.assertEqual(retriever.block_records[0]							,B"This is the first record" )

					# Test mocked remote path
					with patch("transformers.models.realm.retrieval_realm.hf_hub_download" ) as mock_hf_hub_download:
									__A   			= os.path.join(
									    os.path.join(self.tmpdirname							,"realm_block_records" )							,_REALM_BLOCK_RECORDS_FILENAME )
									__A   			= RealmRetriever.from_pretrained("google/realm-cc-news-pretrained-openqa" )

					self.assertEqual(retriever.block_records[0]							,B"This is the first record" )


import re
import string

import numpy as np

import datasets


__lowerCAmelCase			:     Optional[int]           =   '\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

__lowerCAmelCase			:     Optional[int]           =   '\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

__lowerCAmelCase			:     Optional[int]           =   '\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,       _KWARGS_DESCRIPTION     )
class        UpperCAmelCase_			( datasets.Metric     ):


  '''simple docstring'''







  def 							_lowercase							( self			: str    )			->	Optional[int]:







    """simple docstring"""






    return datasets.MetricInfo(
        description=_DESCRIPTION			,							citation=_CITATION			,							inputs_description=_KWARGS_DESCRIPTION			,							features=datasets.Features(
            {
                """predictions""": datasets.Value("""string"""			,							id="""sequence"""    ),
                """references""": datasets.Value("""string"""			,							id="""sequence"""    ),
            }    )			,							reference_urls=[]			,							)





  def 							_lowercase							( self			: Optional[int]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Optional[Any]=None			,							UpperCamelCase__			: Optional[Any]=False			,							UpperCamelCase__			: int=False			,							UpperCamelCase__			: Tuple=False			,							)			->	Dict:







    """simple docstring"""






    if regexes_to_ignore is not None:
      for s in regexes_to_ignore:
        __magic_name__				     =       np.array([re.sub(UpperCamelCase__			,							""""""			,							UpperCamelCase__    ) for x in predictions]    )
        __magic_name__				     =       np.array([re.sub(UpperCamelCase__			,							""""""			,							UpperCamelCase__    ) for x in references]    )
    else:
      __magic_name__				     =       np.asarray(UpperCamelCase__    )
      __magic_name__				     =       np.asarray(UpperCamelCase__    )

    if ignore_case:
      __magic_name__				     =       np.char.lower(UpperCamelCase__    )
      __magic_name__				     =       np.char.lower(UpperCamelCase__    )

    if ignore_punctuation:
      __magic_name__				     =       string.punctuation.maketrans(""""""			,							""""""			,							string.punctuation    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )

    if ignore_numbers:
      __magic_name__				     =       string.digits.maketrans(""""""			,							""""""			,							string.digits    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )

    __magic_name__				     =       predictions == references

    return {"exact_match": np.mean(UpperCamelCase__    ) * 100}

