File size: 3,258 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
datasets = [
[
dict(
abbr='lukaemon_mmlu_high_school_geography',
eval_cfg=dict(
evaluator=dict(
type='opencompass.openicl.icl_evaluator.AccEvaluator')),
infer_cfg=dict(
ice_template=dict(
template=dict(
A=
'{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: A\n',
B=
'{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: B\n',
C=
'{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: C\n',
D='{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: D\n'
),
type=
'opencompass.openicl.icl_prompt_template.PromptTemplate'),
inferencer=dict(
type='opencompass.openicl.icl_inferencer.PPLInferencer'),
prompt_template=dict(
ice_token='</E>',
template=dict(
A=
'The following are multiple choice questions (with answers) about high school geography.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: A',
B=
'The following are multiple choice questions (with answers) about high school geography.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: B',
C=
'The following are multiple choice questions (with answers) about high school geography.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: C',
D='The following are multiple choice questions (with answers) about high school geography.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: D'
),
type=
'opencompass.openicl.icl_prompt_template.PromptTemplate'),
retriever=dict(
fix_id_list=[
0,
1,
2,
3,
4,
],
type='opencompass.openicl.icl_retriever.FixKRetriever')),
name='high_school_geography',
path='./data/mmlu/',
reader_cfg=dict(
input_columns=[
'input',
'A',
'B',
'C',
'D',
],
output_column='target',
train_split='dev'),
type='opencompass.datasets.MMLUDataset'),
],
]
eval = dict(runner=dict(task=dict()))
models = [
dict(
abbr='my_api',
api_key='',
batch_size=8,
max_out_len=100,
max_seq_len=2048,
meta_template=dict(round=[
dict(api_role='HUMAN', role='HUMAN'),
dict(api_role='BOT', generate=True, role='BOT'),
]),
path='my_api',
run_cfg=dict(num_gpus=1, num_procs=1),
type='opencompass.models.my_api.MyAPIModel',
url='http://127.0.0.1:12345/testing'),
]
work_dir = './outputs/default/20240306_164404'
|