File size: 2,203 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
from typing import List, Optional
from mmengine import ConfigDict
from opencompass.utils import dataset_abbr_from_cfg
from opencompass.utils.prompt import get_prompt_hash
from .default import DefaultSummarizer
class CircularSummarizer(DefaultSummarizer):
def __init__(self,
config: ConfigDict,
dataset_abbrs: Optional[List[str]] = None,
summary_groups: List = [],
prompt_db=None,
metric_types=None) -> None:
super().__init__(config, dataset_abbrs, summary_groups, prompt_db)
self.metric_types = metric_types
def _format_table(self, parsed_results, dataset_metrics,
dataset_eval_mode):
prompt_version = {
dataset_abbr_from_cfg(d): get_prompt_hash(d)[:6]
for d in self.dataset_cfgs
}
table = []
header1 = ['dataset', 'version', 'mode'] + sum(
[[model_abbr] + ['-' for _ in range(len(self.metric_types) - 1)]
for model_abbr in self.model_abbrs], [])
table.append(header1)
header2 = ['-', '-', '-'] + sum(
[self.metric_types for _ in self.model_abbrs], [])
table.append(header2)
for dataset_abbr in self.dataset_abbrs:
if dataset_abbr not in dataset_metrics:
table.append([dataset_abbr, '-', '-'] + ['-'] *
len(self.model_abbrs) * len(self.metric_types))
continue
row = [
dataset_abbr,
prompt_version.get(dataset_abbr, '-'),
dataset_eval_mode.get(dataset_abbr, '-')
]
for model_abbr in self.model_abbrs:
for metric in self.metric_types:
if dataset_abbr in parsed_results[
model_abbr] and metric in parsed_results[
model_abbr][dataset_abbr]:
row.append('{:.02f}'.format(
parsed_results[model_abbr][dataset_abbr][metric]))
else:
row.append('-')
table.append(row)
return table
|