# Path: t5/evaluation/metrics.py
# def bleu(targets, predictions):
# def rouge(targets, predictions, score_keys=None):
#   def _prepare_summary(summary):
# def span_squad(targets, predictions):
#   def space_tok(s):
#   def get_answer_text_from_context(context, answer_tokens):
# def squad(targets, predictions):
# def trivia_qa(targets, predictions):
# def accuracy(targets, predictions):
# def sequence_accuracy(targets, predictions):
# def pearson_corrcoef(targets, predictions):
# def spearman_corrcoef(targets, predictions):
# def mean_multiclass_f1(num_classes, **metric_fn_kwargs):
# def all_match(targets, predictions):
# def f1_score_with_invalid(targets, predictions):
# def mean_group_metric(metric_fn,
#                       group_key="group",
#                       value_key="value",
#                       return_subgroup_scores=False):
#   def my_metric(targets, predictions):
# def multirc_f1_over_all_answers(targets, predictions):
# def auc(targets, predictions, targets_threshold=None):
# def score_auc(targets, scores, targets_threshold=None):
# def sklearn_metrics_wrapper(metric_str,
#                             metric_dict_str=None,
#                             metric_post_process_fn=None,
#                             **metric_fn_kwargs):
#   def fn(targets, predictions):
# def rank_classification(
#     targets: Sequence[Tuple[Sequence[int], bool, float, int]],
#     scores: Sequence[float],
#     num_classes: Optional[int] = None,
#     normalize_by_target_length: bool = False,
# ) -> Dict[str, Union[float, int]]:
#   def exp_normalize(x):
# def _coqa_tokenize(inp: str) -> Sequence[str]:
#   def remove_articles(text):
#   def normalize_whitespace(text):
#   def remove_punc(text):
# def _sequence_f1(target_tokens: Sequence[str],
#                  prediction_tokens: Sequence[str]) -> float:
# def coqa_f1(
#     targets: Sequence[Sequence[str]], predictions: Sequence[str]
# ) -> Mapping[str, float]:
# def edit_distance(targets, predictions, lower=True):
# 
# Path: t5/evaluation/test_utils.py
# class BaseMetricsTest(absltest.TestCase):
#   def assertDictClose(self, a, b, delta=None, places=None):


from absl.testing import absltest
from t5.evaluation import metrics
from t5.evaluation import test_utils
import sklearn.metrics
          [
              0.1, 0.5
          ],
          num_classes=2)

    with self.assertRaisesWithLiteralMatch(
        ValueError,
        "The first element of `targets` ('idx') should be 2-dimensional. Got "
        "0."):
      metrics.rank_classification(
          [
              (0, True, 1.0, 1),
              (0, True, 1.0, 1),
          ],
          [
              0.1, 0.5
          ],
          num_classes=2)

  def test_coqa_tokenize(self):
    self.assertEqual(metrics._coqa_tokenize("Maru the cat"), ["maru", "cat"])
    self.assertEqual(metrics._coqa_tokenize("Maru  cat"), ["maru", "cat"])
    self.assertEqual(metrics._coqa_tokenize("Maru the cat."), ["maru", "cat"])

  def test_sequence_f1(self):
    self.assertEqual(metrics._sequence_f1([], []), 1.0)
    self.assertEqual(metrics._sequence_f1([], ["cat"]), 0.0)
    self.assertEqual(metrics._sequence_f1(["cat"], []), 0.0)
    self.assertEqual(metrics._sequence_f1(["dog"], ["cat"]), 0.0)
    self.assertAlmostEqual(metrics._sequence_f1(["cat", "dog"], ["cat"]), 2 / 3)

# Path: t5/evaluation/metrics.py
# def bleu(targets, predictions):
# def rouge(targets, predictions, score_keys=None):
#   def _prepare_summary(summary):
# def span_squad(targets, predictions):
#   def space_tok(s):
#   def get_answer_text_from_context(context, answer_tokens):
# def squad(targets, predictions):
# def trivia_qa(targets, predictions):
# def accuracy(targets, predictions):
# def sequence_accuracy(targets, predictions):
# def pearson_corrcoef(targets, predictions):
# def spearman_corrcoef(targets, predictions):
# def mean_multiclass_f1(num_classes, **metric_fn_kwargs):
# def all_match(targets, predictions):
# def f1_score_with_invalid(targets, predictions):
# def mean_group_metric(metric_fn,
#                       group_key="group",
#                       value_key="value",
#                       return_subgroup_scores=False):
#   def my_metric(targets, predictions):
# def multirc_f1_over_all_answers(targets, predictions):
# def auc(targets, predictions, targets_threshold=None):
# def score_auc(targets, scores, targets_threshold=None):
# def sklearn_metrics_wrapper(metric_str,
#                             metric_dict_str=None,
#                             metric_post_process_fn=None,
#                             **metric_fn_kwargs):
#   def fn(targets, predictions):
# def rank_classification(
#     targets: Sequence[Tuple[Sequence[int], bool, float, int]],
#     scores: Sequence[float],
#     num_classes: Optional[int] = None,
#     normalize_by_target_length: bool = False,
# ) -> Dict[str, Union[float, int]]:
#   def exp_normalize(x):
# def _coqa_tokenize(inp: str) -> Sequence[str]:
#   def remove_articles(text):
#   def normalize_whitespace(text):
#   def remove_punc(text):
# def _sequence_f1(target_tokens: Sequence[str],
#                  prediction_tokens: Sequence[str]) -> float:
# def coqa_f1(
#     targets: Sequence[Sequence[str]], predictions: Sequence[str]
# ) -> Mapping[str, float]:
# def edit_distance(targets, predictions, lower=True):
# 
# Path: t5/evaluation/test_utils.py
# class BaseMetricsTest(absltest.TestCase):
#   def assertDictClose(self, a, b, delta=None, places=None):


# Path: t5/evaluation/metrics_test.py
from absl.testing import absltest
from t5.evaluation import metrics
from t5.evaluation import test_utils
import sklearn.metrics

          [
              0.1, 0.5
          ],
          num_classes=2)

    with self.assertRaisesWithLiteralMatch(
        ValueError,
        "The first element of `targets` ('idx') should be 2-dimensional. Got "
        "0."):
      metrics.rank_classification(
          [
              (0, True, 1.0, 1),
              (0, True, 1.0, 1),
          ],
          [
              0.1, 0.5
          ],
          num_classes=2)

  def test_coqa_tokenize(self):
    self.assertEqual(metrics._coqa_tokenize("Maru the cat"), ["maru", "cat"])
    self.assertEqual(metrics._coqa_tokenize("Maru  cat"), ["maru", "cat"])
    self.assertEqual(metrics._coqa_tokenize("Maru the cat."), ["maru", "cat"])

  def test_sequence_f1(self):
    self.assertEqual(metrics._sequence_f1([], []), 1.0)
    self.assertEqual(metrics._sequence_f1([], ["cat"]), 0.0)
    self.assertEqual(metrics._sequence_f1(["cat"], []), 0.0)
    self.assertEqual(metrics._sequence_f1(["dog"], ["cat"]), 0.0)
    self.assertAlmostEqual(metrics._sequence_f1(["cat", "dog"], ["cat"]), 2 / 3)

    self.assertAlmostEqual(metrics._sequence_f1(["cat"], ["cat", "dog"]), 2 / 3)