
import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_PRETRAINING_MAPPING,
        MobileBertForMaskedLM,
        MobileBertForMultipleChoice,
        MobileBertForNextSentencePrediction,
        MobileBertForPreTraining,
        MobileBertForQuestionAnswering,
        MobileBertForSequenceClassification,
        MobileBertForTokenClassification,
        MobileBertModel,
    )




class        UpperCAmelCase_			:


  '''simple docstring'''







  def __init__( self			: List[str]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: str=13			,							UpperCamelCase__			: List[Any]=7			,							UpperCamelCase__			: Any=True			,							UpperCamelCase__			: List[Any]=True			,							UpperCamelCase__			: Tuple=True			,							UpperCamelCase__			: Union[str, Any]=True			,							UpperCamelCase__			: List[Any]=99			,							UpperCamelCase__			: List[Any]=64			,							UpperCamelCase__			: Optional[Any]=32			,							UpperCamelCase__			: Dict=5			,							UpperCamelCase__			: str=4			,							UpperCamelCase__			: List[str]=37			,							UpperCamelCase__			: List[Any]="gelu"			,							UpperCamelCase__			: str=0.1			,							UpperCamelCase__			: List[Any]=0.1			,							UpperCamelCase__			: Tuple=512			,							UpperCamelCase__			: int=16			,							UpperCamelCase__			: Optional[Any]=2			,							UpperCamelCase__			: Union[str, Any]=0.02			,							UpperCamelCase__			: Optional[int]=3			,							UpperCamelCase__			: Any=4			,							UpperCamelCase__			: int=None			,							)			->	List[Any]:







    """simple docstring"""






    __magic_name__				     =       parent
    __magic_name__				     =       batch_size
    __magic_name__				     =       seq_length
    __magic_name__				     =       is_training
    __magic_name__				     =       use_input_mask
    __magic_name__				     =       use_token_type_ids
    __magic_name__				     =       use_labels
    __magic_name__				     =       vocab_size
    __magic_name__				     =       hidden_size
    __magic_name__				     =       embedding_size
    __magic_name__				     =       num_hidden_layers
    __magic_name__				     =       num_attention_heads
    __magic_name__				     =       intermediate_size
    __magic_name__				     =       hidden_act
    __magic_name__				     =       hidden_dropout_prob
    __magic_name__				     =       attention_probs_dropout_prob
    __magic_name__				     =       max_position_embeddings
    __magic_name__				     =       type_vocab_size
    __magic_name__				     =       type_sequence_label_size
    __magic_name__				     =       initializer_range
    __magic_name__				     =       num_labels
    __magic_name__				     =       num_choices
    __magic_name__				     =       scope
  def 							_lowercase							( self			: Union[str, Any]    )			->	List[str]:







    """simple docstring"""






    __magic_name__				     =       ids_tensor([self.batch_size, self.seq_length]			,							self.vocab_size    )

    __magic_name__				     =       None
    if self.use_input_mask:
      __magic_name__				     =       random_attention_mask([self.batch_size, self.seq_length]    )

    __magic_name__				     =       None
    if self.use_token_type_ids:
      __magic_name__				     =       ids_tensor([self.batch_size, self.seq_length]			,							self.type_vocab_size    )

    __magic_name__				     =       None
    __magic_name__				     =       None
    __magic_name__				     =       None
    if self.use_labels:
      __magic_name__				     =       ids_tensor([self.batch_size]			,							self.type_sequence_label_size    )
      __magic_name__				     =       ids_tensor([self.batch_size, self.seq_length]			,							self.num_labels    )
      __magic_name__				     =       ids_tensor([self.batch_size]			,							self.num_choices    )

    __magic_name__				     =       self.get_config()

    return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  def 							_lowercase							( self			: Dict    )			->	Optional[Any]:







    """simple docstring"""






    return MobileBertConfig(
        vocab_size=self.vocab_size			,							hidden_size=self.hidden_size			,							num_hidden_layers=self.num_hidden_layers			,							num_attention_heads=self.num_attention_heads			,							intermediate_size=self.intermediate_size			,							embedding_size=self.embedding_size			,							hidden_act=self.hidden_act			,							hidden_dropout_prob=self.hidden_dropout_prob			,							attention_probs_dropout_prob=self.attention_probs_dropout_prob			,							max_position_embeddings=self.max_position_embeddings			,							type_vocab_size=self.type_vocab_size			,							is_decoder=UpperCamelCase__			,							initializer_range=self.initializer_range			,							)
  def 							_lowercase							( self			: Any			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: Dict    )			->	Optional[Any]:







    """simple docstring"""






    __magic_name__				     =       MobileBertModel(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__    )
    __magic_name__				     =       model(UpperCamelCase__			,							token_type_ids=UpperCamelCase__    )
    __magic_name__				     =       model(UpperCamelCase__    )

    self.parent.assertEqual(result.last_hidden_state.shape			,							(self.batch_size, self.seq_length, self.hidden_size)    )
    self.parent.assertEqual(result.pooler_output.shape			,							(self.batch_size, self.hidden_size)    )
  def 							_lowercase							( self			: int			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: List[Any]    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       MobileBertForMaskedLM(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__    )
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)    )
  def 							_lowercase							( self			: List[str]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Union[str, Any]    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       MobileBertForNextSentencePrediction(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(
        UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__			,							)
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, 2)    )
  def 							_lowercase							( self			: List[Any]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: str			,							UpperCamelCase__			: str			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Optional[int]    )			->	Dict:







    """simple docstring"""






    __magic_name__				     =       MobileBertForPreTraining(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(
        UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__			,							next_sentence_label=UpperCamelCase__			,							)
    self.parent.assertEqual(result.prediction_logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)    )
    self.parent.assertEqual(result.seq_relationship_logits.shape			,							(self.batch_size, 2)    )
  def 							_lowercase							( self			: List[Any]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Any    )			->	str:







    """simple docstring"""






    __magic_name__				     =       MobileBertForQuestionAnswering(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(
        UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							start_positions=UpperCamelCase__			,							end_positions=UpperCamelCase__			,							)
    self.parent.assertEqual(result.start_logits.shape			,							(self.batch_size, self.seq_length)    )
    self.parent.assertEqual(result.end_logits.shape			,							(self.batch_size, self.seq_length)    )
  def 							_lowercase							( self			: Dict			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: str    )			->	int:







    """simple docstring"""






    __magic_name__				     =       self.num_labels
    __magic_name__				     =       MobileBertForSequenceClassification(UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__    )
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_labels)    )
  def 							_lowercase							( self			: Optional[int]			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Union[str, Any]    )			->	str:







    """simple docstring"""






    __magic_name__				     =       self.num_labels
    __magic_name__				     =       MobileBertForTokenClassification(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__    )
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.num_labels)    )
  def 							_lowercase							( self			: Optional[Any]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: str    )			->	Optional[int]:







    """simple docstring"""






    __magic_name__				     =       self.num_choices
    __magic_name__				     =       MobileBertForMultipleChoice(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       input_ids.unsqueeze(1    ).expand(-1			,							self.num_choices			,							-1    ).contiguous()
    __magic_name__				     =       token_type_ids.unsqueeze(1    ).expand(-1			,							self.num_choices			,							-1    ).contiguous()
    __magic_name__				     =       input_mask.unsqueeze(1    ).expand(-1			,							self.num_choices			,							-1    ).contiguous()
    __magic_name__				     =       model(
        UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__			,							)
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_choices)    )





  def 							_lowercase							( self			: List[Any]    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       self.prepare_config_and_inputs()
    (
        (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    
    )				     =       config_and_inputs
    __magic_name__				     =       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
    return config, inputs_dict







@require_torch
class        UpperCAmelCase_			( _A  ,       _A  ,       unittest.TestCase     ):


  '''simple docstring'''







  a__           =		(
      (
          MobileBertModel,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  a__           =		(
      {
          """feature-extraction""": MobileBertModel,
          """fill-mask""": MobileBertForMaskedLM,
          """question-answering""": MobileBertForQuestionAnswering,
          """text-classification""": MobileBertForSequenceClassification,
          """token-classification""": MobileBertForTokenClassification,
          """zero-shot""": MobileBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  a__           =		True
  def 							_lowercase							( self			: Optional[Any]			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: Any=False    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       super()._prepare_for_class(UpperCamelCase__			,							UpperCamelCase__			,							return_labels=UpperCamelCase__    )

    if return_labels:
      if model_class in get_values(UpperCamelCase__    ):
        __magic_name__				     =       torch.zeros(
            (self.model_tester.batch_size, self.model_tester.seq_length)			,							dtype=torch.long			,							device=UpperCamelCase__    )
        __magic_name__				     =       torch.zeros(
            self.model_tester.batch_size			,							dtype=torch.long			,							device=UpperCamelCase__    )
    return inputs_dict
  def 							_lowercase							( self			: List[Any]    )			->	str:







    """simple docstring"""






    __magic_name__				     =       MobileBertModelTester(self    )
    __magic_name__				     =       ConfigTester(self			,							config_class=UpperCamelCase__			,							hidden_size=37    )
  def 							_lowercase							( self			: Optional[int]    )			->	List[str]:







    """simple docstring"""






    self.config_tester.run_common_tests()
  def 							_lowercase							( self			: List[str]    )			->	Any:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_model(*UpperCamelCase__    )
  def 							_lowercase							( self			: Optional[int]    )			->	Optional[Any]:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_masked_lm(*UpperCamelCase__    )
  def 							_lowercase							( self			: List[str]    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_multiple_choice(*UpperCamelCase__    )
  def 							_lowercase							( self			: Union[str, Any]    )			->	Optional[int]:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*UpperCamelCase__    )
  def 							_lowercase							( self			: int    )			->	int:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_pretraining(*UpperCamelCase__    )
  def 							_lowercase							( self			: List[str]    )			->	List[str]:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_question_answering(*UpperCamelCase__    )
  def 							_lowercase							( self			: Dict    )			->	Dict:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_sequence_classification(*UpperCamelCase__    )





  def 							_lowercase							( self			: Any    )			->	List[str]:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_token_classification(*UpperCamelCase__    )







def a__    (       A_					):






  '''simple docstring'''





  return torch.tensor(
      A_,     dtype=torch.long,     device=A_,     )


__lowerCAmelCase			:     Optional[Any]           =   1e-3





@require_torch
@require_sentencepiece
@require_tokenizers
class        UpperCAmelCase_			( unittest.TestCase     ):


  '''simple docstring'''







  @slow
  def 							_lowercase							( self			: str    )			->	List[str]:







    """simple docstring"""






    __magic_name__				     =       MobileBertModel.from_pretrained("""google/mobilebert-uncased"""    ).to(UpperCamelCase__    )
    __magic_name__				     =       _long_tensor([[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]]    )
    with torch.no_grad():
      __magic_name__				     =       model(UpperCamelCase__    )[0]
    __magic_name__				     =       torch.Size((1, 9, 512)    )
    self.assertEqual(output.shape			,							UpperCamelCase__    )
    __magic_name__				     =       torch.tensor(
        [
            [
                [-2.473_6526E07, 8.269_1656E04, 1.652_1838E05],
                [-5.754_1704E-01, 3.905_6022E00, 4.401_1507E00],
                [2.604_7359E00, 1.567_7652E00, -1.732_4188E-01],
            ]
        ]			,							device=UpperCamelCase__			,							)

    # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
    # ~1 difference, it's therefore not a good idea to measure using addition.
    # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
    # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
    __magic_name__				     =       torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE    )
    __magic_name__				     =       torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE    )

    self.assertTrue(lower_bound and upper_bound    )



import re
import string

import numpy as np

import datasets


__lowerCAmelCase			:     Optional[int]           =   '\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

__lowerCAmelCase			:     Optional[int]           =   '\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

__lowerCAmelCase			:     Optional[int]           =   '\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,       _KWARGS_DESCRIPTION     )
class        UpperCAmelCase_			( datasets.Metric     ):


  '''simple docstring'''







  def 							_lowercase							( self			: str    )			->	Optional[int]:







    """simple docstring"""






    return datasets.MetricInfo(
        description=_DESCRIPTION			,							citation=_CITATION			,							inputs_description=_KWARGS_DESCRIPTION			,							features=datasets.Features(
            {
                """predictions""": datasets.Value("""string"""			,							id="""sequence"""    ),
                """references""": datasets.Value("""string"""			,							id="""sequence"""    ),
            }    )			,							reference_urls=[]			,							)





  def 							_lowercase							( self			: Optional[int]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Optional[Any]=None			,							UpperCamelCase__			: Optional[Any]=False			,							UpperCamelCase__			: int=False			,							UpperCamelCase__			: Tuple=False			,							)			->	Dict:







    """simple docstring"""






    if regexes_to_ignore is not None:
      for s in regexes_to_ignore:
        __magic_name__				     =       np.array([re.sub(UpperCamelCase__			,							""""""			,							UpperCamelCase__    ) for x in predictions]    )
        __magic_name__				     =       np.array([re.sub(UpperCamelCase__			,							""""""			,							UpperCamelCase__    ) for x in references]    )
    else:
      __magic_name__				     =       np.asarray(UpperCamelCase__    )
      __magic_name__				     =       np.asarray(UpperCamelCase__    )

    if ignore_case:
      __magic_name__				     =       np.char.lower(UpperCamelCase__    )
      __magic_name__				     =       np.char.lower(UpperCamelCase__    )

    if ignore_punctuation:
      __magic_name__				     =       string.punctuation.maketrans(""""""			,							""""""			,							string.punctuation    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )

    if ignore_numbers:
      __magic_name__				     =       string.digits.maketrans(""""""			,							""""""			,							string.digits    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )
      __magic_name__				     =       np.char.translate(UpperCamelCase__			,							table=UpperCamelCase__    )

    __magic_name__				     =       predictions == references

    return {"exact_match": np.mean(UpperCamelCase__    ) * 100}

