# Path: t5/evaluation/eval_utils.py
# class Metric(object):
#   def __init__(self, name, group=None):
# def parse_events_files(tb_summary_dir, seqio_summaries=False):
# def get_eval_metric_values(events, task_name=None):
# def sort_columns(df, metric_names=None):
# def compute_avg_glue(df, metric_names=None):
# def scores_to_df(scores, metric_names=None):
# def metric_group_max(df, metric_names=None):
# def log_csv(df, metric_names=None, output_file=None):
# METRIC_NAMES = collections.OrderedDict([
#     ("glue_average", Metric("Average GLUE Score")),
#     ("glue_cola_v002/matthews_corrcoef", Metric("CoLA")),
#     ("glue_sst2_v002/accuracy", Metric("SST-2")),
#     ("glue_mrpc_v002/f1", Metric("MRPC (F1)", "MRPC")),
#     ("glue_mrpc_v002/accuracy", Metric("MRPC (accuracy)", "MRPC")),
#     ("glue_stsb_v002/pearson_corrcoef", Metric("STSB (Pearson)", "STSB")),
#     ("glue_stsb_v002/spearman_corrcoef", Metric("STSB (Spearman)", "STSB")),
#     ("glue_qqp_v002/f1", Metric("QQP (F1)", "QQP")),
#     ("glue_qqp_v002/accuracy", Metric("QQP (accuracy)", "QQP")),
#     ("glue_mnli_matched_v002/accuracy", Metric("MNLIm", "MNLI")),
#     ("glue_mnli_mismatched_v002/accuracy", Metric("MNLImm", "MNLI")),
#     ("glue_qnli_v002/accuracy", Metric("QNLI")),
#     ("glue_rte_v002/accuracy", Metric("GLUE RTE")),
#     ("cnn_dailymail_v002/rouge1", Metric("CNN/DM (ROUGE-1)", "CNN/DM")),
#     ("cnn_dailymail_v002/rouge2", Metric("CNN/DM (ROUGE-2)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeL", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeLsum", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("squad_v010_allanswers/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010_allanswers_span/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers_span/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("super_glue_average", Metric("Average SuperGLUE Score")),
#     ("super_glue_boolq_v102/accuracy", Metric("BoolQ (accuracy)")),
#     ("super_glue_cb_v102/mean_3class_f1", Metric("CB (F1)", "CB")),
#     ("super_glue_cb_v102/accuracy", Metric("CB (accuracy)", "CB")),
#     ("super_glue_copa_v102/accuracy", Metric("CoPA")),
#     ("super_glue_multirc_v102/f1", Metric("MultiRC (F1)", "MultiRC")),
#     ("super_glue_multirc_v102/exact_match", Metric("MultiRC (EM)", "MultiRC")),
#     ("super_glue_record_v102/f1", Metric("ReCoRD (F1)", "ReCoRD")),
#     ("super_glue_record_v102/em", Metric("ReCoRD (EM)", "ReCoRD")),
#     ("super_glue_rte_v102/accuracy", Metric("SuperGLUE RTE")),
#     ("super_glue_wic_v102/accuracy", Metric("WiC")),
#     ("super_glue_wsc_v102_simple_eval/accuracy", Metric("WSC")),
#     ("dpr_v001_simple/accuracy", Metric("DPR")),
#     ("wmt_t2t_ende_v003/bleu", Metric("WMT T2T En-De")),
#     ("wmt14_ende_v003/bleu", Metric("WMT14 En-De")),
#     ("wmt15_enfr_v003/bleu", Metric("WMT15 En-Fr")),
#     ("wmt16_enro_v003/bleu", Metric("WMT16 En-Ro")),
# ])


import os
import tensorflow as tf
from absl import app
from absl import flags
from absl import logging
from t5.evaluation import eval_utils
flags.DEFINE_bool("seqio_summaries", False, "Whether summaries are generated "
                  "from SeqIO Evaluator.")


def main(_):
  if FLAGS.seqio_summaries:
    subdirs = tf.io.gfile.listdir(FLAGS.summary_dir)
    summary_dirs = [os.path.join(FLAGS.summary_dir, d) for d in subdirs]
  else:
    summary_dirs = [FLAGS.summary_dir]

  scores = None
  for d in summary_dirs:
    events = eval_utils.parse_events_files(d, FLAGS.seqio_summaries)
    if FLAGS.perplexity_eval:
      task_metrics = events
    else:
      task_metrics = eval_utils.get_eval_metric_values(
          events,
          task_name=os.path.basename(d) if FLAGS.seqio_summaries else None)
    if scores:
      scores.update(task_metrics)
    else:
      scores = task_metrics

  if not scores:
    logging.info("No evaluation events found in %s", FLAGS.summary_dir)
    return
  df = eval_utils.scores_to_df(scores)
  df = eval_utils.compute_avg_glue(df)

# Path: t5/evaluation/eval_utils.py
# class Metric(object):
#   def __init__(self, name, group=None):
# def parse_events_files(tb_summary_dir, seqio_summaries=False):
# def get_eval_metric_values(events, task_name=None):
# def sort_columns(df, metric_names=None):
# def compute_avg_glue(df, metric_names=None):
# def scores_to_df(scores, metric_names=None):
# def metric_group_max(df, metric_names=None):
# def log_csv(df, metric_names=None, output_file=None):
# METRIC_NAMES = collections.OrderedDict([
#     ("glue_average", Metric("Average GLUE Score")),
#     ("glue_cola_v002/matthews_corrcoef", Metric("CoLA")),
#     ("glue_sst2_v002/accuracy", Metric("SST-2")),
#     ("glue_mrpc_v002/f1", Metric("MRPC (F1)", "MRPC")),
#     ("glue_mrpc_v002/accuracy", Metric("MRPC (accuracy)", "MRPC")),
#     ("glue_stsb_v002/pearson_corrcoef", Metric("STSB (Pearson)", "STSB")),
#     ("glue_stsb_v002/spearman_corrcoef", Metric("STSB (Spearman)", "STSB")),
#     ("glue_qqp_v002/f1", Metric("QQP (F1)", "QQP")),
#     ("glue_qqp_v002/accuracy", Metric("QQP (accuracy)", "QQP")),
#     ("glue_mnli_matched_v002/accuracy", Metric("MNLIm", "MNLI")),
#     ("glue_mnli_mismatched_v002/accuracy", Metric("MNLImm", "MNLI")),
#     ("glue_qnli_v002/accuracy", Metric("QNLI")),
#     ("glue_rte_v002/accuracy", Metric("GLUE RTE")),
#     ("cnn_dailymail_v002/rouge1", Metric("CNN/DM (ROUGE-1)", "CNN/DM")),
#     ("cnn_dailymail_v002/rouge2", Metric("CNN/DM (ROUGE-2)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeL", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("cnn_dailymail_v002/rougeLsum", Metric("CNN/DM (ROUGE-L)", "CNN/DM")),
#     ("squad_v010_allanswers/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010_allanswers_span/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010_allanswers_span/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("squad_v010/em", Metric("SQuAD (EM)", "SQuAD")),
#     ("squad_v010/f1", Metric("SQuAD (F1)", "SQuAD")),
#     ("super_glue_average", Metric("Average SuperGLUE Score")),
#     ("super_glue_boolq_v102/accuracy", Metric("BoolQ (accuracy)")),
#     ("super_glue_cb_v102/mean_3class_f1", Metric("CB (F1)", "CB")),
#     ("super_glue_cb_v102/accuracy", Metric("CB (accuracy)", "CB")),
#     ("super_glue_copa_v102/accuracy", Metric("CoPA")),
#     ("super_glue_multirc_v102/f1", Metric("MultiRC (F1)", "MultiRC")),
#     ("super_glue_multirc_v102/exact_match", Metric("MultiRC (EM)", "MultiRC")),
#     ("super_glue_record_v102/f1", Metric("ReCoRD (F1)", "ReCoRD")),
#     ("super_glue_record_v102/em", Metric("ReCoRD (EM)", "ReCoRD")),
#     ("super_glue_rte_v102/accuracy", Metric("SuperGLUE RTE")),
#     ("super_glue_wic_v102/accuracy", Metric("WiC")),
#     ("super_glue_wsc_v102_simple_eval/accuracy", Metric("WSC")),
#     ("dpr_v001_simple/accuracy", Metric("DPR")),
#     ("wmt_t2t_ende_v003/bleu", Metric("WMT T2T En-De")),
#     ("wmt14_ende_v003/bleu", Metric("WMT14 En-De")),
#     ("wmt15_enfr_v003/bleu", Metric("WMT15 En-Fr")),
#     ("wmt16_enro_v003/bleu", Metric("WMT16 En-Ro")),
# ])


# Path: t5/scripts/parse_tb.py
import os
import tensorflow as tf
from absl import app
from absl import flags
from absl import logging
from t5.evaluation import eval_utils

flags.DEFINE_bool("seqio_summaries", False, "Whether summaries are generated "
                  "from SeqIO Evaluator.")


def main(_):
  if FLAGS.seqio_summaries:
    subdirs = tf.io.gfile.listdir(FLAGS.summary_dir)
    summary_dirs = [os.path.join(FLAGS.summary_dir, d) for d in subdirs]
  else:
    summary_dirs = [FLAGS.summary_dir]

  scores = None
  for d in summary_dirs:
    events = eval_utils.parse_events_files(d, FLAGS.seqio_summaries)
    if FLAGS.perplexity_eval:
      task_metrics = events
    else:
      task_metrics = eval_utils.get_eval_metric_values(
          events,
          task_name=os.path.basename(d) if FLAGS.seqio_summaries else None)
    if scores:
      scores.update(task_metrics)
    else:
      scores = task_metrics

  if not scores:
    logging.info("No evaluation events found in %s", FLAGS.summary_dir)
    return
  df = eval_utils.scores_to_df(scores)
  df = eval_utils.compute_avg_glue(df)

  df = eval_utils.sort_columns(df)