File size: 1,552 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
import os
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.datasets import GaokaoBenchDataset
from mmengine.config import read_base
with read_base():
from .GaokaoBench_prompts import MCQ_prompts, FBQ_prompts
GaokaoBench_datasets = []
for folder, prompts in [
("Multiple-choice_Questions", MCQ_prompts),
("Fill-in-the-blank_Questions", FBQ_prompts),
]:
for p in prompts:
reader_cfg = {
"input_columns": ["question"],
"output_column": "answer",
}
infer_cfg = {
"ice_template": {
"type": PromptTemplate,
"template": {"round": [{"role": "HUMAN", "prompt": p["prefix_prompt"] + "{question}"}]},
"ice_token": "</E>",
},
"retriever": {"type": ZeroRetriever},
"inferencer": {"type": GenInferencer, "max_out_len": 1024},
}
eval_cfg = {
"evaluator": {"type": "GaokaoBenchEvaluator" + "_" + p["type"]},
"pred_role": "BOT",
}
dataset = {
"type": GaokaoBenchDataset,
"abbr": "GaokaoBench_" + p["keyword"],
"path": os.path.join("data", "GAOKAO-BENCH", "data", folder, p["keyword"] + ".json"),
"reader_cfg": reader_cfg,
"infer_cfg": infer_cfg,
"eval_cfg": eval_cfg,
}
GaokaoBench_datasets.append(dataset)
|