|
import json |
|
import os.path as osp |
|
|
|
from datasets import Dataset |
|
|
|
from opencompass.openicl.icl_evaluator import BaseEvaluator |
|
from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET |
|
|
|
from .base import BaseDataset |
|
|
|
|
|
@LOAD_DATASET.register_module() |
|
class OpenFinDataDataset(BaseDataset): |
|
|
|
@staticmethod |
|
def load(path: str, name: str): |
|
with open(osp.join(path, f'{name}.json'), 'r') as f: |
|
data = json.load(f) |
|
return Dataset.from_list(data) |
|
|
|
|
|
@ICL_EVALUATORS.register_module() |
|
class OpenFinDataKWEvaluator(BaseEvaluator): |
|
|
|
def __init__(self, ): |
|
super().__init__() |
|
|
|
def score(self, predictions, references): |
|
assert len(predictions) == len(references) |
|
|
|
scores = [] |
|
results = dict() |
|
|
|
for i in range(len(references)): |
|
all_hit = True |
|
judgement = references[i].split('、') |
|
for item in judgement: |
|
if item not in predictions[i]: |
|
all_hit = False |
|
break |
|
if all_hit: |
|
scores.append(True) |
|
else: |
|
scores.append(False) |
|
|
|
results['accuracy'] = round(sum(scores) / len(scores), 4) * 100 |
|
return results |
|
|
