File size: 4,018 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
datasets = [
[
dict(
abbr='ceval-computer_network',
eval_cfg=dict(
evaluator=dict(
type='opencompass.openicl.icl_evaluator.AccEvaluator')),
infer_cfg=dict(
ice_template=dict(
ice_token='</E>',
template=dict(
A=dict(
begin='</E>',
round=[
dict(
prompt=
'以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ',
role='HUMAN'),
dict(prompt='A', role='BOT'),
]),
B=dict(
begin='</E>',
round=[
dict(
prompt=
'以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ',
role='HUMAN'),
dict(prompt='B', role='BOT'),
]),
C=dict(
begin='</E>',
round=[
dict(
prompt=
'以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ',
role='HUMAN'),
dict(prompt='C', role='BOT'),
]),
D=dict(
begin='</E>',
round=[
dict(
prompt=
'以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ',
role='HUMAN'),
dict(prompt='D', role='BOT'),
])),
type=
'opencompass.openicl.icl_prompt_template.PromptTemplate'),
inferencer=dict(
type='opencompass.openicl.icl_inferencer.PPLInferencer'),
retriever=dict(
fix_id_list=[
0,
1,
2,
3,
4,
],
type='opencompass.openicl.icl_retriever.FixKRetriever')),
name='computer_network',
path='./data/ceval/formal_ceval',
reader_cfg=dict(
input_columns=[
'question',
'A',
'B',
'C',
'D',
],
output_column='answer',
test_split='val',
train_split='dev'),
type='opencompass.datasets.CEvalDataset'),
],
]
eval = dict(runner=dict(task=dict()))
models = [
dict(
abbr='llama-7b-hf',
batch_padding=False,
batch_size=8,
max_out_len=100,
max_seq_len=2048,
model_kwargs=dict(device_map='auto'),
path='huggyllama/llama-7b',
run_cfg=dict(num_gpus=1, num_procs=1),
tokenizer_kwargs=dict(
padding_side='left', truncation_side='left', use_fast=False),
tokenizer_path='huggyllama/llama-7b',
type='opencompass.models.HuggingFaceCausalLM'),
]
work_dir = './outputs/default/20240304_175403'
|