# Path: t5/evaluation/eval_utils.py
# class Metric(object):
#   def __init__(self, name, group=None):
# def parse_events_files(tb_summary_dir, seqio_summaries=False):
# def get_eval_metric_values(events, task_name=None):
# def sort_columns(df, metric_names=None):
# def compute_avg_glue(df, metric_names=None):
# def scores_to_df(scores, metric_names=None):
# def metric_group_max(df, metric_names=None):
# def log_csv(df, metric_names=None, output_file=None):
# METRIC_NAMES = collections.OrderedDict([
#     ("glue_average", Metric("Average GLUE Score")),
#     ("glue_cola_v002/matthews_corrcoef", Metric("CoLA")),
#     ("glue_sst2_v002/accuracy", Metric("SST-2")),
#     ("glue_mrpc_v002/f1", Metric("MRPC (F1)", "MRPC")),
#     ("glue_mrpc_v002/accuracy", Metric("MRPC (accuracy)", "MRPC")),
#     ("glue_stsb_v002/pearson_corrcoef", Metric("STSB (Pearson)", "STSB")),
#     ("glue_stsb_v002/spearman_corrcoef", Metric("STSB (Spearman)", "STSB")),
#     ("glue_qqp_v002/f1", Metric("QQP (F1)", "QQP")),
#     ("glue_qqp_v002/accuracy", Metric("QQP (accuracy)", "QQP")),
#     ("glue_mnli_matched_v002/accuracy", Metric("MNLIm", "MNLI")),
#     ("glue_mnli_mismatched_v002/accuracy", Metric("MNLImm", "MNLI")),
#     ("glue_qnli_v002/accuracy", Metric("QNLI")),
#     ("glue_rte_v002/accuracy", Metric("GLUE RTE")),
#     ("cnn_dailymail_v002/rouge1", Metric("CNN/DM (ROUGE-1)", "CNN/DM")),
#     ("cnn_dailymail_v002/rouge2", Metric("CNN/DM (ROUGE-2)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeL", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeLsum", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("squad_v010_allanswers/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010_allanswers_span/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers_span/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("super_glue_average", Metric("Average SuperGLUE Score")),
#     ("super_glue_boolq_v102/accuracy", Metric("BoolQ (accuracy)")),
#     ("super_glue_cb_v102/mean_3class_f1", Metric("CB (F1)", "CB")),
#     ("super_glue_cb_v102/accuracy", Metric("CB (accuracy)", "CB")),
#     ("super_glue_copa_v102/accuracy", Metric("CoPA")),
#     ("super_glue_multirc_v102/f1", Metric("MultiRC (F1)", "MultiRC")),
#     ("super_glue_multirc_v102/exact_match", Metric("MultiRC (EM)", "MultiRC")),
#     ("super_glue_record_v102/f1", Metric("ReCoRD (F1)", "ReCoRD")),
#     ("super_glue_record_v102/em", Metric("ReCoRD (EM)", "ReCoRD")),
#     ("super_glue_rte_v102/accuracy", Metric("SuperGLUE RTE")),
#     ("super_glue_wic_v102/accuracy", Metric("WiC")),
#     ("super_glue_wsc_v102_simple_eval/accuracy", Metric("WSC")),
#     ("dpr_v001_simple/accuracy", Metric("DPR")),
#     ("wmt_t2t_ende_v003/bleu", Metric("WMT T2T En-De")),
#     ("wmt14_ende_v003/bleu", Metric("WMT14 En-De")),
#     ("wmt15_enfr_v003/bleu", Metric("WMT15 En-Fr")),
#     ("wmt16_enro_v003/bleu", Metric("WMT16 En-Ro")),
# ])


import collections
import os
import numpy as np
import pandas as pd
import seqio
import tensorflow.compat.v1 as tf
from absl.testing import absltest
from t5.evaluation import eval_utils
    )/8.
    self.assertSequenceAlmostEqual(
        df["Average SuperGLUE Score"], [expected_super, 2*expected_super]
    )
    del df["CoLA"]
    del df["Average GLUE Score"]
    df = eval_utils.compute_avg_glue(df)
    self.assertNoCommonElements(df.columns, ["Average GLUE Score"])

  def test_metric_group_max(self):
    df = pd.DataFrame(
        collections.OrderedDict([
            ("ABC Accuracy", [1., 2., 3., 4.]),
            ("DEF Exact Match", [0., 10., 3., 0.]),
            ("DEF Accuracy", [4., 7., 8., 0.]),
        ]),
        index=[10, 20, 30, 40],
    )
    metric_names = collections.OrderedDict([
        ("metric1", eval_utils.Metric("ABC Accuracy")),
        ("metric2", eval_utils.Metric("DEF Accuracy", "DEF")),
        ("metric3", eval_utils.Metric("DEF Exact Match", "DEF")),
    ])
    metric_max, metric_max_step = eval_utils.metric_group_max(df, metric_names)
    self.assertTrue(metric_max.keys().equals(df.columns))
    self.assertListEqual(list(metric_max.values), [4., 10., 7.])
    self.assertTrue(metric_max_step.keys().equals(df.columns))
    self.assertSequenceEqual(list(metric_max_step.values), [40, 20, 20])

  def test_log_csv(self):

# Path: t5/evaluation/eval_utils.py
# class Metric(object):
#   def __init__(self, name, group=None):
# def parse_events_files(tb_summary_dir, seqio_summaries=False):
# def get_eval_metric_values(events, task_name=None):
# def sort_columns(df, metric_names=None):
# def compute_avg_glue(df, metric_names=None):
# def scores_to_df(scores, metric_names=None):
# def metric_group_max(df, metric_names=None):
# def log_csv(df, metric_names=None, output_file=None):
# METRIC_NAMES = collections.OrderedDict([
#     ("glue_average", Metric("Average GLUE Score")),
#     ("glue_cola_v002/matthews_corrcoef", Metric("CoLA")),
#     ("glue_sst2_v002/accuracy", Metric("SST-2")),
#     ("glue_mrpc_v002/f1", Metric("MRPC (F1)", "MRPC")),
#     ("glue_mrpc_v002/accuracy", Metric("MRPC (accuracy)", "MRPC")),
#     ("glue_stsb_v002/pearson_corrcoef", Metric("STSB (Pearson)", "STSB")),
#     ("glue_stsb_v002/spearman_corrcoef", Metric("STSB (Spearman)", "STSB")),
#     ("glue_qqp_v002/f1", Metric("QQP (F1)", "QQP")),
#     ("glue_qqp_v002/accuracy", Metric("QQP (accuracy)", "QQP")),
#     ("glue_mnli_matched_v002/accuracy", Metric("MNLIm", "MNLI")),
#     ("glue_mnli_mismatched_v002/accuracy", Metric("MNLImm", "MNLI")),
#     ("glue_qnli_v002/accuracy", Metric("QNLI")),
#     ("glue_rte_v002/accuracy", Metric("GLUE RTE")),
#     ("cnn_dailymail_v002/rouge1", Metric("CNN/DM (ROUGE-1)", "CNN/DM")),
#     ("cnn_dailymail_v002/rouge2", Metric("CNN/DM (ROUGE-2)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeL", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeLsum", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("squad_v010_allanswers/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010_allanswers_span/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers_span/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("super_glue_average", Metric("Average SuperGLUE Score")),
#     ("super_glue_boolq_v102/accuracy", Metric("BoolQ (accuracy)")),
#     ("super_glue_cb_v102/mean_3class_f1", Metric("CB (F1)", "CB")),
#     ("super_glue_cb_v102/accuracy", Metric("CB (accuracy)", "CB")),
#     ("super_glue_copa_v102/accuracy", Metric("CoPA")),
#     ("super_glue_multirc_v102/f1", Metric("MultiRC (F1)", "MultiRC")),
#     ("super_glue_multirc_v102/exact_match", Metric("MultiRC (EM)", "MultiRC")),
#     ("super_glue_record_v102/f1", Metric("ReCoRD (F1)", "ReCoRD")),
#     ("super_glue_record_v102/em", Metric("ReCoRD (EM)", "ReCoRD")),
#     ("super_glue_rte_v102/accuracy", Metric("SuperGLUE RTE")),
#     ("super_glue_wic_v102/accuracy", Metric("WiC")),
#     ("super_glue_wsc_v102_simple_eval/accuracy", Metric("WSC")),
#     ("dpr_v001_simple/accuracy", Metric("DPR")),
#     ("wmt_t2t_ende_v003/bleu", Metric("WMT T2T En-De")),
#     ("wmt14_ende_v003/bleu", Metric("WMT14 En-De")),
#     ("wmt15_enfr_v003/bleu", Metric("WMT15 En-Fr")),
#     ("wmt16_enro_v003/bleu", Metric("WMT16 En-Ro")),
# ])


# Path: t5/evaluation/eval_utils_test.py
import collections
import os
import numpy as np
import pandas as pd
import seqio
import tensorflow.compat.v1 as tf
from absl.testing import absltest
from t5.evaluation import eval_utils

    )/8.
    self.assertSequenceAlmostEqual(
        df["Average SuperGLUE Score"], [expected_super, 2*expected_super]
    )
    del df["CoLA"]
    del df["Average GLUE Score"]
    df = eval_utils.compute_avg_glue(df)
    self.assertNoCommonElements(df.columns, ["Average GLUE Score"])

  def test_metric_group_max(self):
    df = pd.DataFrame(
        collections.OrderedDict([
            ("ABC Accuracy", [1., 2., 3., 4.]),
            ("DEF Exact Match", [0., 10., 3., 0.]),
            ("DEF Accuracy", [4., 7., 8., 0.]),
        ]),
        index=[10, 20, 30, 40],
    )
    metric_names = collections.OrderedDict([
        ("metric1", eval_utils.Metric("ABC Accuracy")),
        ("metric2", eval_utils.Metric("DEF Accuracy", "DEF")),
        ("metric3", eval_utils.Metric("DEF Exact Match", "DEF")),
    ])
    metric_max, metric_max_step = eval_utils.metric_group_max(df, metric_names)
    self.assertTrue(metric_max.keys().equals(df.columns))
    self.assertListEqual(list(metric_max.values), [4., 10., 7.])
    self.assertTrue(metric_max_step.keys().equals(df.columns))
    self.assertSequenceEqual(list(metric_max_step.values), [40, 20, 20])

  def test_log_csv(self):

    metric_names = list(eval_utils.METRIC_NAMES.values())