File size: 1,190 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
import csv
import os.path as osp
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET
from .base import BaseDataset
@LOAD_DATASET.register_module()
class FinanceIQDataset(BaseDataset):
# @staticmethod
# def load(path: str):
# from datasets import load_dataset
# return load_dataset('csv', data_files={'test': path})
@staticmethod
def load(path: str, name: str):
dataset = DatasetDict()
for split in ['dev', 'test']:
raw_data = []
filename = osp.join(path, split, f'{name}.csv')
with open(filename, encoding='utf-8') as f:
reader = csv.reader(f)
_ = next(reader) # skip the header
for row in reader:
assert len(row) == 7
raw_data.append({
'question': row[1],
'A': row[2],
'B': row[3],
'C': row[4],
'D': row[5],
'answer': row[6],
})
dataset[split] = Dataset.from_list(raw_data)
return dataset
|