| datasets = [ | |
| [ | |
| dict( | |
| abbr='siqa', | |
| eval_cfg=dict( | |
| evaluator=dict( | |
| type='opencompass.openicl.icl_evaluator.EDAccEvaluator'), | |
| pred_role='BOT'), | |
| infer_cfg=dict( | |
| inferencer=dict( | |
| type='opencompass.openicl.icl_inferencer.GenInferencer'), | |
| prompt_template=dict( | |
| template=dict(round=[ | |
| dict( | |
| prompt= | |
| '{context}\nQuestion: {question}\nA. {answerA}\nB. {answerB}\nC. {answerC}\nAnswer:', | |
| role='HUMAN'), | |
| ]), | |
| type= | |
| 'opencompass.openicl.icl_prompt_template.PromptTemplate'), | |
| retriever=dict( | |
| type='opencompass.openicl.icl_retriever.ZeroRetriever')), | |
| path='./data/siqa', | |
| reader_cfg=dict( | |
| input_columns=[ | |
| 'context', | |
| 'question', | |
| 'answerA', | |
| 'answerB', | |
| 'answerC', | |
| ], | |
| output_column='all_labels', | |
| test_split='validation'), | |
| type='opencompass.datasets.siqaDataset_V2'), | |
| ], | |
| ] | |
| eval = dict(runner=dict(task=dict())) | |
| models = [ | |
| dict( | |
| abbr='my_api', | |
| batch_size=8, | |
| key='', | |
| max_out_len=100, | |
| max_seq_len=2048, | |
| meta_template=dict(round=[ | |
| dict(begin='<reserved_106>', role='HUMAN'), | |
| dict(begin='<reserved_107>', generate=True, role='BOT'), | |
| ]), | |
| model_kwargs=dict(device_map='auto', trust_remote_code=True), | |
| path='http://127.0.0.1:12345/testing', | |
| run_cfg=dict(num_gpus=1, num_procs=1), | |
| type='opencompass.models.my_api.MyAPIModel'), | |
| ] | |
| work_dir = './outputs/default/20240306_161832' | |