File size: 1,686 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import PPLOnlyInferencer
from opencompass.openicl.icl_evaluator import AveragePPLEvaluator
from opencompass.datasets import SanitizedMBPPDataset, JsonlDataset
mbpp_datasets = []
mbpp_infer_cfg = dict(
prompt_template=dict(type=PromptTemplate, template="{text}\n{code}"),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLOnlyInferencer),
)
mbpp_eval_cfg = dict(evaluator=dict(type=AveragePPLEvaluator))
for split in ['train', 'test']:
mbpp_reader_cfg = dict(
input_columns=['text', 'code'],
output_column=None,
train_split=split,
test_split=split,
)
mbpp_datasets.append(
dict(
abbr=f'mbpp-{split}-ppl',
type=SanitizedMBPPDataset,
path='./data/mbpp/sanitized-mbpp.jsonl',
reader_cfg=mbpp_reader_cfg,
infer_cfg=mbpp_infer_cfg,
eval_cfg=mbpp_eval_cfg)
)
mbpp_infer_cfg = dict(
prompt_template=dict(type=PromptTemplate, template="{text}"),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLOnlyInferencer),
)
mbpp_eval_cfg = dict(evaluator=dict(type=AveragePPLEvaluator))
mbpp_reader_cfg = dict(
input_columns=['text'],
output_column=None,
)
mbpp_datasets.append(
dict(
abbr=f'mbpp-ref-ppl',
type=JsonlDataset,
path='/mnt/petrelfs/zhoufengzhe/repos/cscripts/mock-datas/mock_mbpp_20240113.jsonl',
reader_cfg=mbpp_reader_cfg,
infer_cfg=mbpp_infer_cfg,
eval_cfg=mbpp_eval_cfg
)
)
|