File size: 2,263 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
from copy import deepcopy
_base_summary_groups = [
{
'name': 'teval-instruct_v1',
'metric': 'format_metric',
'subsets': [
['teval-instruct_v1', 'string_format_metric'],
['teval-instruct_v1', 'json_format_metric'],
]
},
{
'name': 'teval-instruct_v1',
'metric': 'args_em_metric',
'subsets': [
['teval-instruct_v1', 'string_args_em_metric'],
['teval-instruct_v1', 'json_args_em_metric'],
]
},
{
'name': 'teval-instruct_v1',
'metric': 'string_metric',
'subsets': [
['teval-instruct_v1', 'string_format_metric'],
['teval-instruct_v1', 'string_args_em_metric'],
]
},
{
'name': 'teval-instruct_v1',
'metric': 'json_metric',
'subsets': [
['teval-instruct_v1', 'json_format_metric'],
['teval-instruct_v1', 'json_args_em_metric'],
]
},
{
'name': 'copy_teval-review_str_v1',
'subsets': [
['teval-review_str_v1', 'review_quality'],
],
},
{
'name': 'teval',
'subsets': [
['teval-instruct_v1', 'format_metric'],
['teval-instruct_v1', 'args_em_metric'],
['teval-plan_str_v1', 'f1_score'],
['teval-plan_json_v1', 'f1_score'],
['teval-reason_str_v1', 'thought'],
['teval-reason_retrieve_understand_json_v1', 'thought'],
['teval-retrieve_str_v1', 'name'],
['teval-reason_retrieve_understand_json_v1', 'name'],
['teval-understand_str_v1', 'args'],
['teval-reason_retrieve_understand_json_v1', 'args'],
['teval-review_str_v1', 'review_quality'],
['copy_teval-review_str_v1', 'naive_average'], # a hack for review * 2
]
},
]
teval_summary_groups = []
# base
for group in _base_summary_groups:
group = deepcopy(group)
teval_summary_groups.append(group)
# base _zh
for group in _base_summary_groups:
group = deepcopy(group)
group['name'] = group['name'] + '_zh'
group['subsets'] = [[subset[0] + '_zh', subset[1]] for subset in group['subsets']]
teval_summary_groups.append(group)
|