File size: 1,555 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 |
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import PPLOnlyInferencer
from opencompass.openicl.icl_evaluator import AveragePPLEvaluator
from opencompass.datasets import JsonlDataset
mmlu_datasets = []
mmlu_infer_cfg = dict(
prompt_template=dict(type=PromptTemplate, template="{text}"),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLOnlyInferencer),
)
mmlu_eval_cfg = dict(evaluator=dict(type=AveragePPLEvaluator))
mmlu_reader_cfg = dict(
input_columns=['text'],
output_column=None,
)
mmlu_datasets.append(
dict(
abbr=f'mmlu-test-ppl',
type=JsonlDataset,
path='/mnt/petrelfs/zhoufengzhe/repos/cscripts/mock-datas/mmlu_test_content.jsonl',
reader_cfg=mmlu_reader_cfg,
infer_cfg=mmlu_infer_cfg,
eval_cfg=mmlu_eval_cfg
)
)
mmlu_infer_cfg = dict(
prompt_template=dict(type=PromptTemplate, template="{rephrase}"),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLOnlyInferencer),
)
mmlu_eval_cfg = dict(evaluator=dict(type=AveragePPLEvaluator))
mmlu_reader_cfg = dict(
input_columns=['rephrase'],
output_column=None,
)
mmlu_datasets.append(
dict(
abbr=f'mmlu-ref-ppl',
type=JsonlDataset,
path='/mnt/petrelfs/zhoufengzhe/repos/cscripts/mock-datas/mmlu_test_content.jsonl',
reader_cfg=mmlu_reader_cfg,
infer_cfg=mmlu_infer_cfg,
eval_cfg=mmlu_eval_cfg
)
)
|