





import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        FlaubertForMultipleChoice,
        FlaubertForQuestionAnswering,
        FlaubertForQuestionAnsweringSimple,
        FlaubertForSequenceClassification,
        FlaubertForTokenClassification,
        FlaubertModel,
        FlaubertWithLMHeadModel,
    )
    from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class  A__    (							__UpperCAmelCase	):




   """simple docstring"""



   def __init__(					self					,  lowercase					,  lowercase=13					,  lowercase=7					,  lowercase=True					,  lowercase=True					,  lowercase=True					,  lowercase=True					,  lowercase=True					,  lowercase=False					,  lowercase=False					,  lowercase=False					,  lowercase=2					,  lowercase=99					,  lowercase=0					,  lowercase=32					,  lowercase=5					,  lowercase=4					,  lowercase=0.1					,  lowercase=0.1					,  lowercase=512					,  lowercase=12					,  lowercase=2					,  lowercase=0.02					,  lowercase=3					,  lowercase=4					,  lowercase="last"					,  lowercase=None					,  lowercase=None					,  )			->						Any:







          '''simple docstring'''

          a__						:	Optional[Any] =				parent
          a__						:	Dict =				batch_size
          a__						:	int =				seq_length
          a__						:	str =				is_training
          a__						:	Any =				use_input_lengths
          a__						:	Any =				use_token_type_ids
          a__						:	Union[str, Any] =				use_labels
          a__						:	int =				gelu_activation
          a__						:	Optional[Any] =				sinusoidal_embeddings
          a__						:	int =				causal
          a__						:	Optional[Any] =				asm
          a__						:	str =				n_langs
          a__						:	List[str] =				vocab_size
          a__						:	Union[str, Any] =				n_special
          a__						:	Optional[int] =				hidden_size
          a__						:	Union[str, Any] =				num_hidden_layers
          a__						:	Dict =				num_attention_heads
          a__						:	Union[str, Any] =				hidden_dropout_prob
          a__						:	Optional[Any] =				attention_probs_dropout_prob
          a__						:	Optional[Any] =				max_position_embeddings
          a__						:	int =				type_vocab_size
          a__						:	Optional[Any] =				type_sequence_label_size
          a__						:	Optional[Any] =				initializer_range
          a__						:	Tuple =				num_labels
          a__						:	Union[str, Any] =				num_choices
          a__						:	Tuple =				summary_type
          a__						:	List[Any] =				use_proj
          a__						:	Union[str, Any] =				scope

   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          a__						:	Optional[Any] =				ids_tensor([self.batch_size, self.seq_length]					,  self.vocab_size)
          a__						:	Optional[int] =				random_attention_mask([self.batch_size, self.seq_length])

          a__						:	List[str] =				None
          if self.use_input_lengths:
                 a__						:	Optional[Any] =				(
                     ids_tensor([self.batch_size]					,  vocab_size=2) + self.seq_length - 2
                 )  # small variation of seq_length

          a__						:	List[str] =				None
          if self.use_token_type_ids:
                 a__						:	Optional[Any] =				ids_tensor([self.batch_size, self.seq_length]					,  self.n_langs)

          a__						:	List[str] =				None
          a__						:	Tuple =				None
          a__						:	Optional[Any] =				None
          if self.use_labels:
                 a__						:	List[Any] =				ids_tensor([self.batch_size]					,  self.type_sequence_label_size)
                 a__						:	List[str] =				ids_tensor([self.batch_size, self.seq_length]					,  self.num_labels)
                 a__						:	Union[str, Any] =				ids_tensor([self.batch_size]					,  2).float()
                 a__						:	Union[str, Any] =				ids_tensor([self.batch_size]					,  self.num_choices)

          a__						:	List[str] =				self.get_config()

          return (
              config,
              input_ids,
              token_type_ids,
              input_lengths,
              sequence_labels,
              token_labels,
              is_impossible_labels,
              choice_labels,
              input_mask,
          )

   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          return FlaubertConfig(
              vocab_size=self.vocab_size					,  n_special=self.n_special					,  emb_dim=self.hidden_size					,  n_layers=self.num_hidden_layers					,  n_heads=self.num_attention_heads					,  dropout=self.hidden_dropout_prob					,  attention_dropout=self.attention_probs_dropout_prob					,  gelu_activation=self.gelu_activation					,  sinusoidal_embeddings=self.sinusoidal_embeddings					,  asm=self.asm					,  causal=self.causal					,  n_langs=self.n_langs					,  max_position_embeddings=self.max_position_embeddings					,  initializer_range=self.initializer_range					,  summary_type=self.summary_type					,  use_proj=self.use_proj					,  )

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  )			->						List[str]:







          '''simple docstring'''

          a__						:	Optional[int] =				FlaubertModel(config=lowercase)
          model.to(lowercase)
          model.eval()
          a__						:	Optional[int] =				model(lowercase					,  lengths=lowercase					,  langs=lowercase)
          a__						:	Tuple =				model(lowercase					,  langs=lowercase)
          a__						:	Union[str, Any] =				model(lowercase)
          self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size))

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  )			->						Optional[Any]:







          '''simple docstring'''

          a__						:	Dict =				FlaubertWithLMHeadModel(lowercase)
          model.to(lowercase)
          model.eval()

          a__						:	int =				model(lowercase					,  token_type_ids=lowercase					,  labels=lowercase)
          self.parent.assertEqual(result.loss.shape					,  ())
          self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.vocab_size))

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  )			->						Optional[int]:







          '''simple docstring'''

          a__						:	Optional[Any] =				FlaubertForQuestionAnsweringSimple(lowercase)
          model.to(lowercase)
          model.eval()

          a__						:	Optional[Any] =				model(lowercase)

          a__						:	Dict =				model(lowercase					,  start_positions=lowercase					,  end_positions=lowercase)
          self.parent.assertEqual(result.start_logits.shape					,  (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape					,  (self.batch_size, self.seq_length))

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  )			->						Optional[Any]:







          '''simple docstring'''

          a__						:	str =				FlaubertForQuestionAnswering(lowercase)
          model.to(lowercase)
          model.eval()

          a__						:	Dict =				model(lowercase)

          a__						:	Optional[int] =				model(
              lowercase					,  start_positions=lowercase					,  end_positions=lowercase					,  cls_index=lowercase					,  is_impossible=lowercase					,  p_mask=lowercase					,  )

          a__						:	Optional[int] =				model(
              lowercase					,  start_positions=lowercase					,  end_positions=lowercase					,  cls_index=lowercase					,  is_impossible=lowercase					,  )

          ((a__) ,							)						:	int =				result_with_labels.to_tuple()

          a__						:	Any =				model(lowercase					,  start_positions=lowercase					,  end_positions=lowercase)

          ((a__) ,							)						:	Optional[Any] =				result_with_labels.to_tuple()

          self.parent.assertEqual(result_with_labels.loss.shape					,  ())
          self.parent.assertEqual(result.start_top_log_probs.shape					,  (self.batch_size, model.config.start_n_top))
          self.parent.assertEqual(result.start_top_index.shape					,  (self.batch_size, model.config.start_n_top))
          self.parent.assertEqual(
              result.end_top_log_probs.shape					,  (self.batch_size, model.config.start_n_top * model.config.end_n_top))
          self.parent.assertEqual(
              result.end_top_index.shape					,  (self.batch_size, model.config.start_n_top * model.config.end_n_top))
          self.parent.assertEqual(result.cls_logits.shape					,  (self.batch_size,))

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  )			->						List[str]:







          '''simple docstring'''

          a__						:	Optional[int] =				FlaubertForSequenceClassification(lowercase)
          model.to(lowercase)
          model.eval()

          a__						:	List[Any] =				model(lowercase)
          a__						:	Optional[Any] =				model(lowercase					,  labels=lowercase)

          self.parent.assertEqual(result.loss.shape					,  ())
          self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.type_sequence_label_size))

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  )			->						List[Any]:







          '''simple docstring'''

          a__						:	Dict =				self.num_labels
          a__						:	Any =				FlaubertForTokenClassification(lowercase)
          model.to(lowercase)
          model.eval()

          a__						:	List[str] =				model(lowercase					,  attention_mask=lowercase					,  labels=lowercase)
          self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.num_labels))

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  )			->						Optional[Any]:







          '''simple docstring'''

          a__						:	Optional[int] =				self.num_choices
          a__						:	List[Any] =				FlaubertForMultipleChoice(config=lowercase)
          model.to(lowercase)
          model.eval()
          a__						:	Tuple =				input_ids.unsqueeze(1).expand(-1					,  self.num_choices					,  -1).contiguous()
          a__						:	str =				token_type_ids.unsqueeze(1).expand(-1					,  self.num_choices					,  -1).contiguous()
          a__						:	Optional[int] =				input_mask.unsqueeze(1).expand(-1					,  self.num_choices					,  -1).contiguous()
          a__						:	Tuple =				model(
              lowercase					,  attention_mask=lowercase					,  token_type_ids=lowercase					,  labels=lowercase					,  )
          self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_choices))

   def 							__lowercase		(					self)			->						Optional[int]:







          '''simple docstring'''

          a__						:	Optional[int] =				self.prepare_config_and_inputs()
          (
              (
              a__
          ) ,							(
              a__
          ) ,							(
              a__
          ) ,							(
              a__
          ) ,							(
              a__
          ) ,							(
              a__
          ) ,							(
              a__
          ) ,							(
              a__
          ) ,							(
              a__
          ) ,							
          )						:	Optional[Any] =				config_and_inputs
          a__						:	Any =				{
              'input_ids': input_ids,
              'token_type_ids': token_type_ids,
              'lengths': input_lengths,
              'attention_mask': input_mask,
          }
          return config, inputs_dict





@require_torch
class  A__    (							__UpperCAmelCase ,  __UpperCAmelCase ,  unittest.TestCase	):




   """simple docstring"""



   __A   :				Dict         =  (
       (
           FlaubertModel,
           FlaubertWithLMHeadModel,
           FlaubertForQuestionAnswering,
           FlaubertForQuestionAnsweringSimple,
           FlaubertForSequenceClassification,
           FlaubertForTokenClassification,
           FlaubertForMultipleChoice,
       )
       if is_torch_available()
       else ()
   )
   __A   :				List[Any]         =  (
       {
           '''feature-extraction''': FlaubertModel,
           '''fill-mask''': FlaubertWithLMHeadModel,
           '''question-answering''': FlaubertForQuestionAnsweringSimple,
           '''text-classification''': FlaubertForSequenceClassification,
           '''token-classification''': FlaubertForTokenClassification,
           '''zero-shot''': FlaubertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase					,  lowercase					,  lowercase)			->						Optional[int]:







          '''simple docstring'''

          if (
              pipeline_test_casse_name == "QAPipelineTests"
              and tokenizer_name is not None
              and not tokenizer_name.endswith('Fast')
          ):
                 # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                 # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                 # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                 return True

          return False

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase=False)			->						int:







          '''simple docstring'''

          a__						:	Union[str, Any] =				super()._prepare_for_class(lowercase					,  lowercase					,  return_labels=lowercase)

          if return_labels:
                 if model_class.__name__ == "FlaubertForQuestionAnswering":
                        a__						:	Dict =				torch.zeros(
                            self.model_tester.batch_size					,  dtype=torch.long					,  device=lowercase)
                        a__						:	str =				torch.zeros(
                            self.model_tester.batch_size					,  dtype=torch.long					,  device=lowercase)

          return inputs_dict

   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          a__						:	Union[str, Any] =				FlaubertModelTester(self)
          a__						:	List[str] =				ConfigTester(self					,  config_class=lowercase					,  emb_dim=37)

   def 							__lowercase		(					self)			->						Any:







          '''simple docstring'''

          self.config_tester.run_common_tests()

   def 							__lowercase		(					self)			->						int:







          '''simple docstring'''

          a__						:	Tuple =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_flaubert_model(*lowercase)

   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          a__						:	List[Any] =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_flaubert_lm_head(*lowercase)

   def 							__lowercase		(					self)			->						List[str]:







          '''simple docstring'''

          a__						:	Union[str, Any] =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_flaubert_simple_qa(*lowercase)

   def 							__lowercase		(					self)			->						Dict:







          '''simple docstring'''

          a__						:	List[str] =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_flaubert_qa(*lowercase)

   def 							__lowercase		(					self)			->						Union[str, Any]:







          '''simple docstring'''

          a__						:	Dict =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_flaubert_sequence_classif(*lowercase)

   def 							__lowercase		(					self)			->						Optional[int]:







          '''simple docstring'''

          a__						:	Union[str, Any] =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_flaubert_token_classif(*lowercase)

   def 							__lowercase		(					self)			->						Any:







          '''simple docstring'''

          a__						:	str =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_flaubert_multiple_choice(*lowercase)

   @slow
   def 							__lowercase		(					self)			->						Optional[Any]:







          '''simple docstring'''

          for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 a__						:	Tuple =				FlaubertModel.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)

   @slow
   @require_torch_gpu
   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          a__ ,							a__						:	Union[str, Any] =				self.model_tester.prepare_config_and_inputs_for_common()
          for model_class in self.all_model_classes:
                 # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
                 if model_class == FlaubertForMultipleChoice:
                        return

                 a__						:	Optional[int] =				True
                 a__						:	Any =				model_class(config=lowercase)

                 a__						:	Any =				self._prepare_for_class(lowercase					,  lowercase)
                 a__						:	Any =				torch.jit.trace(
                     lowercase					,  (inputs_dict['input_ids'].to('cpu'), inputs_dict['attention_mask'].to('cpu')))

                 with tempfile.TemporaryDirectory() as tmp:
                        torch.jit.save(lowercase					,  os.path.join(lowercase					,  'traced_model.pt'))
                        a__						:	Tuple =				torch.jit.load(os.path.join(lowercase					,  'traced_model.pt')					,  map_location=lowercase)
                        loaded(inputs_dict['input_ids'].to(lowercase)					,  inputs_dict['attention_mask'].to(lowercase))





@require_torch
class  A__    (							unittest.TestCase	):




   """simple docstring"""



   @slow
   def 							__lowercase		(					self)			->						Optional[int]:







          '''simple docstring'''

          a__						:	List[str] =				FlaubertModel.from_pretrained('flaubert/flaubert_base_cased')
          a__						:	Union[str, Any] =				torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
          with torch.no_grad():
                 a__						:	str =				model(lowercase)[0]
          a__						:	Dict =				torch.Size((1, 11, 768))
          self.assertEqual(output.shape					,  lowercase)
          a__						:	Optional[Any] =				torch.tensor(
              [[[-2.62_51, -1.42_98, -0.02_27], [-2.85_10, -1.63_87, 0.22_58], [-2.81_14, -1.18_32, -0.30_66]]])

          self.assertTrue(torch.allclose(output[:, :3, :3]					,  lowercase					,  atol=1e-4))









from typing import List

import jiwer
import jiwer.transforms as tr
from packaging import version

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse("""3.8"""):
    import importlib_metadata
else:
    import importlib.metadata as importlib_metadata


lowercase      :       Tuple									=  """"""


if version.parse(importlib_metadata.version("""jiwer""")) < version.parse("""2.3.0"""):





    class  A__    (							tr.AbstractTransform	):




       """simple docstring"""



       def __init__(					self					,  lowercase = " ")			->						Tuple:







              '''simple docstring'''

              a__						:	Tuple =				sentence_delimiter

       def 							__lowercase		(					self					,  lowercase)			->						Optional[int]:







              '''simple docstring'''

              return list(lowercase)

       def 							__lowercase		(					self					,  lowercase)			->						Dict:







              '''simple docstring'''

              a__						:	Tuple =				[]
              for sent_idx, sentence in enumerate(lowercase):
                     chars.extend(self.process_string(lowercase))
                     if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(lowercase) - 1:
                            chars.append(self.sentence_delimiter)
              return chars

    lowercase      :       Union[str, Any]									=  tr.Compose(
        [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
    )
else:
    lowercase      :       List[str]									=  tr.Compose(
        [
            tr.RemoveMultipleSpaces(),
            tr.Strip(),
            tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
            tr.ReduceToListOfListOfChars(),
        ]
    )


lowercase      :       List[Any]									=  """\
@inproceedings{inproceedings,
    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},
    year = {2004},
    month = {01},
    pages = {},
    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}
}
"""

lowercase      :       Optional[int]									=  """\
Character error rate (CER) is a common metric of the performance of an automatic speech recognition system.

CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information.

Character error rate can be computed as:

CER = (S + D + I) / N = (S + D + I) / (S + D + C)

where

S is the number of substitutions,
D is the number of deletions,
I is the number of insertions,
C is the number of correct characters,
N is the number of characters in the reference (N=S+D+C).

CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the
performance of the ASR system with a CER of 0 being a perfect score.
"""

lowercase      :       Optional[Any]									=  """
Computes CER score of transcribed segments against references.
Args:
    references: list of references for each speech input.
    predictions: list of transcribtions to score.
    concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result.
Returns:
    (float): the character error rate

Examples:

    >>> predictions = [\"this is the prediction\", \"there is an other sample\"]
    >>> references = [\"this is the reference\", \"there is another one\"]
    >>> cer = datasets.load_metric(\"cer\")
    >>> cer_score = cer.compute(predictions=predictions, references=references)
    >>> print(cer_score)
    0.34146341463414637
"""





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,  _KWARGS_DESCRIPTION	)
class  A__    (							datasets.Metric	):




   """simple docstring"""



   def 							__lowercase		(					self)			->						Union[str, Any]:







          '''simple docstring'''

          return datasets.MetricInfo(
              description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(
                  {
                      'predictions': datasets.Value('string'					,  id='sequence'),
                      'references': datasets.Value('string'					,  id='sequence'),
                  })					,  codebase_urls=['https://github.com/jitsi/jiwer/']					,  reference_urls=[
                  'https://en.wikipedia.org/wiki/Word_error_rate',
                  'https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates',
              ]					,  )

   def 							__lowercase		(					self					,  lowercase					,  lowercase					,  lowercase=False)			->						Any:







          '''simple docstring'''

          if concatenate_texts:
                 return jiwer.compute_measures(
                     lowercase					,  lowercase					,  truth_transform=lowercase					,  hypothesis_transform=lowercase					,  )["wer"]

          a__						:	Optional[int] =				0
          a__						:	str =				0
          for prediction, reference in zip(lowercase					,  lowercase):
                 a__						:	Optional[int] =				jiwer.compute_measures(
                     lowercase					,  lowercase					,  truth_transform=lowercase					,  hypothesis_transform=lowercase					,  )
                 incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
                 total += measures["substitutions"] + measures["deletions"] + measures["hits"]

          return incorrect / total


